xref: /openbmc/linux/net/core/dev.c (revision 8ffdff6a)
1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /*
3  *      NET3    Protocol independent device support routines.
4  *
5  *	Derived from the non IP parts of dev.c 1.0.19
6  *              Authors:	Ross Biro
7  *				Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
8  *				Mark Evans, <evansmp@uhura.aston.ac.uk>
9  *
10  *	Additional Authors:
11  *		Florian la Roche <rzsfl@rz.uni-sb.de>
12  *		Alan Cox <gw4pts@gw4pts.ampr.org>
13  *		David Hinds <dahinds@users.sourceforge.net>
14  *		Alexey Kuznetsov <kuznet@ms2.inr.ac.ru>
15  *		Adam Sulmicki <adam@cfar.umd.edu>
16  *              Pekka Riikonen <priikone@poesidon.pspt.fi>
17  *
18  *	Changes:
19  *              D.J. Barrow     :       Fixed bug where dev->refcnt gets set
20  *                                      to 2 if register_netdev gets called
21  *                                      before net_dev_init & also removed a
22  *                                      few lines of code in the process.
23  *		Alan Cox	:	device private ioctl copies fields back.
24  *		Alan Cox	:	Transmit queue code does relevant
25  *					stunts to keep the queue safe.
26  *		Alan Cox	:	Fixed double lock.
27  *		Alan Cox	:	Fixed promisc NULL pointer trap
28  *		????????	:	Support the full private ioctl range
29  *		Alan Cox	:	Moved ioctl permission check into
30  *					drivers
31  *		Tim Kordas	:	SIOCADDMULTI/SIOCDELMULTI
32  *		Alan Cox	:	100 backlog just doesn't cut it when
33  *					you start doing multicast video 8)
34  *		Alan Cox	:	Rewrote net_bh and list manager.
35  *              Alan Cox        :       Fix ETH_P_ALL echoback lengths.
36  *		Alan Cox	:	Took out transmit every packet pass
37  *					Saved a few bytes in the ioctl handler
38  *		Alan Cox	:	Network driver sets packet type before
39  *					calling netif_rx. Saves a function
40  *					call a packet.
41  *		Alan Cox	:	Hashed net_bh()
42  *		Richard Kooijman:	Timestamp fixes.
43  *		Alan Cox	:	Wrong field in SIOCGIFDSTADDR
44  *		Alan Cox	:	Device lock protection.
45  *              Alan Cox        :       Fixed nasty side effect of device close
46  *					changes.
47  *		Rudi Cilibrasi	:	Pass the right thing to
48  *					set_mac_address()
49  *		Dave Miller	:	32bit quantity for the device lock to
50  *					make it work out on a Sparc.
51  *		Bjorn Ekwall	:	Added KERNELD hack.
52  *		Alan Cox	:	Cleaned up the backlog initialise.
53  *		Craig Metz	:	SIOCGIFCONF fix if space for under
54  *					1 device.
55  *	    Thomas Bogendoerfer :	Return ENODEV for dev_open, if there
56  *					is no device open function.
57  *		Andi Kleen	:	Fix error reporting for SIOCGIFCONF
58  *	    Michael Chastain	:	Fix signed/unsigned for SIOCGIFCONF
59  *		Cyrus Durgin	:	Cleaned for KMOD
60  *		Adam Sulmicki   :	Bug Fix : Network Device Unload
61  *					A network device unload needs to purge
62  *					the backlog queue.
63  *	Paul Rusty Russell	:	SIOCSIFNAME
64  *              Pekka Riikonen  :	Netdev boot-time settings code
65  *              Andrew Morton   :       Make unregister_netdevice wait
66  *                                      indefinitely on dev->refcnt
67  *              J Hadi Salim    :       - Backlog queue sampling
68  *				        - netif_rx() feedback
69  */
70 
71 #include <linux/uaccess.h>
72 #include <linux/bitops.h>
73 #include <linux/capability.h>
74 #include <linux/cpu.h>
75 #include <linux/types.h>
76 #include <linux/kernel.h>
77 #include <linux/hash.h>
78 #include <linux/slab.h>
79 #include <linux/sched.h>
80 #include <linux/sched/mm.h>
81 #include <linux/mutex.h>
82 #include <linux/rwsem.h>
83 #include <linux/string.h>
84 #include <linux/mm.h>
85 #include <linux/socket.h>
86 #include <linux/sockios.h>
87 #include <linux/errno.h>
88 #include <linux/interrupt.h>
89 #include <linux/if_ether.h>
90 #include <linux/netdevice.h>
91 #include <linux/etherdevice.h>
92 #include <linux/ethtool.h>
93 #include <linux/skbuff.h>
94 #include <linux/kthread.h>
95 #include <linux/bpf.h>
96 #include <linux/bpf_trace.h>
97 #include <net/net_namespace.h>
98 #include <net/sock.h>
99 #include <net/busy_poll.h>
100 #include <linux/rtnetlink.h>
101 #include <linux/stat.h>
102 #include <net/dsa.h>
103 #include <net/dst.h>
104 #include <net/dst_metadata.h>
105 #include <net/gro.h>
106 #include <net/pkt_sched.h>
107 #include <net/pkt_cls.h>
108 #include <net/checksum.h>
109 #include <net/xfrm.h>
110 #include <linux/highmem.h>
111 #include <linux/init.h>
112 #include <linux/module.h>
113 #include <linux/netpoll.h>
114 #include <linux/rcupdate.h>
115 #include <linux/delay.h>
116 #include <net/iw_handler.h>
117 #include <asm/current.h>
118 #include <linux/audit.h>
119 #include <linux/dmaengine.h>
120 #include <linux/err.h>
121 #include <linux/ctype.h>
122 #include <linux/if_arp.h>
123 #include <linux/if_vlan.h>
124 #include <linux/ip.h>
125 #include <net/ip.h>
126 #include <net/mpls.h>
127 #include <linux/ipv6.h>
128 #include <linux/in.h>
129 #include <linux/jhash.h>
130 #include <linux/random.h>
131 #include <trace/events/napi.h>
132 #include <trace/events/net.h>
133 #include <trace/events/skb.h>
134 #include <linux/inetdevice.h>
135 #include <linux/cpu_rmap.h>
136 #include <linux/static_key.h>
137 #include <linux/hashtable.h>
138 #include <linux/vmalloc.h>
139 #include <linux/if_macvlan.h>
140 #include <linux/errqueue.h>
141 #include <linux/hrtimer.h>
142 #include <linux/netfilter_ingress.h>
143 #include <linux/crash_dump.h>
144 #include <linux/sctp.h>
145 #include <net/udp_tunnel.h>
146 #include <linux/net_namespace.h>
147 #include <linux/indirect_call_wrapper.h>
148 #include <net/devlink.h>
149 #include <linux/pm_runtime.h>
150 #include <linux/prandom.h>
151 
152 #include "net-sysfs.h"
153 
154 #define MAX_GRO_SKBS 8
155 
156 /* This should be increased if a protocol with a bigger head is added. */
157 #define GRO_MAX_HEAD (MAX_HEADER + 128)
158 
159 static DEFINE_SPINLOCK(ptype_lock);
160 static DEFINE_SPINLOCK(offload_lock);
161 struct list_head ptype_base[PTYPE_HASH_SIZE] __read_mostly;
162 struct list_head ptype_all __read_mostly;	/* Taps */
163 static struct list_head offload_base __read_mostly;
164 
165 static int netif_rx_internal(struct sk_buff *skb);
166 static int call_netdevice_notifiers_info(unsigned long val,
167 					 struct netdev_notifier_info *info);
168 static int call_netdevice_notifiers_extack(unsigned long val,
169 					   struct net_device *dev,
170 					   struct netlink_ext_ack *extack);
171 static struct napi_struct *napi_by_id(unsigned int napi_id);
172 
173 /*
174  * The @dev_base_head list is protected by @dev_base_lock and the rtnl
175  * semaphore.
176  *
177  * Pure readers hold dev_base_lock for reading, or rcu_read_lock()
178  *
179  * Writers must hold the rtnl semaphore while they loop through the
180  * dev_base_head list, and hold dev_base_lock for writing when they do the
181  * actual updates.  This allows pure readers to access the list even
182  * while a writer is preparing to update it.
183  *
184  * To put it another way, dev_base_lock is held for writing only to
185  * protect against pure readers; the rtnl semaphore provides the
186  * protection against other writers.
187  *
188  * See, for example usages, register_netdevice() and
189  * unregister_netdevice(), which must be called with the rtnl
190  * semaphore held.
191  */
192 DEFINE_RWLOCK(dev_base_lock);
193 EXPORT_SYMBOL(dev_base_lock);
194 
195 static DEFINE_MUTEX(ifalias_mutex);
196 
197 /* protects napi_hash addition/deletion and napi_gen_id */
198 static DEFINE_SPINLOCK(napi_hash_lock);
199 
200 static unsigned int napi_gen_id = NR_CPUS;
201 static DEFINE_READ_MOSTLY_HASHTABLE(napi_hash, 8);
202 
203 static DECLARE_RWSEM(devnet_rename_sem);
204 
205 static inline void dev_base_seq_inc(struct net *net)
206 {
207 	while (++net->dev_base_seq == 0)
208 		;
209 }
210 
211 static inline struct hlist_head *dev_name_hash(struct net *net, const char *name)
212 {
213 	unsigned int hash = full_name_hash(net, name, strnlen(name, IFNAMSIZ));
214 
215 	return &net->dev_name_head[hash_32(hash, NETDEV_HASHBITS)];
216 }
217 
218 static inline struct hlist_head *dev_index_hash(struct net *net, int ifindex)
219 {
220 	return &net->dev_index_head[ifindex & (NETDEV_HASHENTRIES - 1)];
221 }
222 
223 static inline void rps_lock(struct softnet_data *sd)
224 {
225 #ifdef CONFIG_RPS
226 	spin_lock(&sd->input_pkt_queue.lock);
227 #endif
228 }
229 
230 static inline void rps_unlock(struct softnet_data *sd)
231 {
232 #ifdef CONFIG_RPS
233 	spin_unlock(&sd->input_pkt_queue.lock);
234 #endif
235 }
236 
237 static struct netdev_name_node *netdev_name_node_alloc(struct net_device *dev,
238 						       const char *name)
239 {
240 	struct netdev_name_node *name_node;
241 
242 	name_node = kmalloc(sizeof(*name_node), GFP_KERNEL);
243 	if (!name_node)
244 		return NULL;
245 	INIT_HLIST_NODE(&name_node->hlist);
246 	name_node->dev = dev;
247 	name_node->name = name;
248 	return name_node;
249 }
250 
251 static struct netdev_name_node *
252 netdev_name_node_head_alloc(struct net_device *dev)
253 {
254 	struct netdev_name_node *name_node;
255 
256 	name_node = netdev_name_node_alloc(dev, dev->name);
257 	if (!name_node)
258 		return NULL;
259 	INIT_LIST_HEAD(&name_node->list);
260 	return name_node;
261 }
262 
263 static void netdev_name_node_free(struct netdev_name_node *name_node)
264 {
265 	kfree(name_node);
266 }
267 
268 static void netdev_name_node_add(struct net *net,
269 				 struct netdev_name_node *name_node)
270 {
271 	hlist_add_head_rcu(&name_node->hlist,
272 			   dev_name_hash(net, name_node->name));
273 }
274 
275 static void netdev_name_node_del(struct netdev_name_node *name_node)
276 {
277 	hlist_del_rcu(&name_node->hlist);
278 }
279 
280 static struct netdev_name_node *netdev_name_node_lookup(struct net *net,
281 							const char *name)
282 {
283 	struct hlist_head *head = dev_name_hash(net, name);
284 	struct netdev_name_node *name_node;
285 
286 	hlist_for_each_entry(name_node, head, hlist)
287 		if (!strcmp(name_node->name, name))
288 			return name_node;
289 	return NULL;
290 }
291 
292 static struct netdev_name_node *netdev_name_node_lookup_rcu(struct net *net,
293 							    const char *name)
294 {
295 	struct hlist_head *head = dev_name_hash(net, name);
296 	struct netdev_name_node *name_node;
297 
298 	hlist_for_each_entry_rcu(name_node, head, hlist)
299 		if (!strcmp(name_node->name, name))
300 			return name_node;
301 	return NULL;
302 }
303 
304 int netdev_name_node_alt_create(struct net_device *dev, const char *name)
305 {
306 	struct netdev_name_node *name_node;
307 	struct net *net = dev_net(dev);
308 
309 	name_node = netdev_name_node_lookup(net, name);
310 	if (name_node)
311 		return -EEXIST;
312 	name_node = netdev_name_node_alloc(dev, name);
313 	if (!name_node)
314 		return -ENOMEM;
315 	netdev_name_node_add(net, name_node);
316 	/* The node that holds dev->name acts as a head of per-device list. */
317 	list_add_tail(&name_node->list, &dev->name_node->list);
318 
319 	return 0;
320 }
321 EXPORT_SYMBOL(netdev_name_node_alt_create);
322 
323 static void __netdev_name_node_alt_destroy(struct netdev_name_node *name_node)
324 {
325 	list_del(&name_node->list);
326 	netdev_name_node_del(name_node);
327 	kfree(name_node->name);
328 	netdev_name_node_free(name_node);
329 }
330 
331 int netdev_name_node_alt_destroy(struct net_device *dev, const char *name)
332 {
333 	struct netdev_name_node *name_node;
334 	struct net *net = dev_net(dev);
335 
336 	name_node = netdev_name_node_lookup(net, name);
337 	if (!name_node)
338 		return -ENOENT;
339 	/* lookup might have found our primary name or a name belonging
340 	 * to another device.
341 	 */
342 	if (name_node == dev->name_node || name_node->dev != dev)
343 		return -EINVAL;
344 
345 	__netdev_name_node_alt_destroy(name_node);
346 
347 	return 0;
348 }
349 EXPORT_SYMBOL(netdev_name_node_alt_destroy);
350 
351 static void netdev_name_node_alt_flush(struct net_device *dev)
352 {
353 	struct netdev_name_node *name_node, *tmp;
354 
355 	list_for_each_entry_safe(name_node, tmp, &dev->name_node->list, list)
356 		__netdev_name_node_alt_destroy(name_node);
357 }
358 
359 /* Device list insertion */
360 static void list_netdevice(struct net_device *dev)
361 {
362 	struct net *net = dev_net(dev);
363 
364 	ASSERT_RTNL();
365 
366 	write_lock_bh(&dev_base_lock);
367 	list_add_tail_rcu(&dev->dev_list, &net->dev_base_head);
368 	netdev_name_node_add(net, dev->name_node);
369 	hlist_add_head_rcu(&dev->index_hlist,
370 			   dev_index_hash(net, dev->ifindex));
371 	write_unlock_bh(&dev_base_lock);
372 
373 	dev_base_seq_inc(net);
374 }
375 
376 /* Device list removal
377  * caller must respect a RCU grace period before freeing/reusing dev
378  */
379 static void unlist_netdevice(struct net_device *dev)
380 {
381 	ASSERT_RTNL();
382 
383 	/* Unlink dev from the device chain */
384 	write_lock_bh(&dev_base_lock);
385 	list_del_rcu(&dev->dev_list);
386 	netdev_name_node_del(dev->name_node);
387 	hlist_del_rcu(&dev->index_hlist);
388 	write_unlock_bh(&dev_base_lock);
389 
390 	dev_base_seq_inc(dev_net(dev));
391 }
392 
393 /*
394  *	Our notifier list
395  */
396 
397 static RAW_NOTIFIER_HEAD(netdev_chain);
398 
399 /*
400  *	Device drivers call our routines to queue packets here. We empty the
401  *	queue in the local softnet handler.
402  */
403 
404 DEFINE_PER_CPU_ALIGNED(struct softnet_data, softnet_data);
405 EXPORT_PER_CPU_SYMBOL(softnet_data);
406 
407 #ifdef CONFIG_LOCKDEP
408 /*
409  * register_netdevice() inits txq->_xmit_lock and sets lockdep class
410  * according to dev->type
411  */
412 static const unsigned short netdev_lock_type[] = {
413 	 ARPHRD_NETROM, ARPHRD_ETHER, ARPHRD_EETHER, ARPHRD_AX25,
414 	 ARPHRD_PRONET, ARPHRD_CHAOS, ARPHRD_IEEE802, ARPHRD_ARCNET,
415 	 ARPHRD_APPLETLK, ARPHRD_DLCI, ARPHRD_ATM, ARPHRD_METRICOM,
416 	 ARPHRD_IEEE1394, ARPHRD_EUI64, ARPHRD_INFINIBAND, ARPHRD_SLIP,
417 	 ARPHRD_CSLIP, ARPHRD_SLIP6, ARPHRD_CSLIP6, ARPHRD_RSRVD,
418 	 ARPHRD_ADAPT, ARPHRD_ROSE, ARPHRD_X25, ARPHRD_HWX25,
419 	 ARPHRD_PPP, ARPHRD_CISCO, ARPHRD_LAPB, ARPHRD_DDCMP,
420 	 ARPHRD_RAWHDLC, ARPHRD_TUNNEL, ARPHRD_TUNNEL6, ARPHRD_FRAD,
421 	 ARPHRD_SKIP, ARPHRD_LOOPBACK, ARPHRD_LOCALTLK, ARPHRD_FDDI,
422 	 ARPHRD_BIF, ARPHRD_SIT, ARPHRD_IPDDP, ARPHRD_IPGRE,
423 	 ARPHRD_PIMREG, ARPHRD_HIPPI, ARPHRD_ASH, ARPHRD_ECONET,
424 	 ARPHRD_IRDA, ARPHRD_FCPP, ARPHRD_FCAL, ARPHRD_FCPL,
425 	 ARPHRD_FCFABRIC, ARPHRD_IEEE80211, ARPHRD_IEEE80211_PRISM,
426 	 ARPHRD_IEEE80211_RADIOTAP, ARPHRD_PHONET, ARPHRD_PHONET_PIPE,
427 	 ARPHRD_IEEE802154, ARPHRD_VOID, ARPHRD_NONE};
428 
429 static const char *const netdev_lock_name[] = {
430 	"_xmit_NETROM", "_xmit_ETHER", "_xmit_EETHER", "_xmit_AX25",
431 	"_xmit_PRONET", "_xmit_CHAOS", "_xmit_IEEE802", "_xmit_ARCNET",
432 	"_xmit_APPLETLK", "_xmit_DLCI", "_xmit_ATM", "_xmit_METRICOM",
433 	"_xmit_IEEE1394", "_xmit_EUI64", "_xmit_INFINIBAND", "_xmit_SLIP",
434 	"_xmit_CSLIP", "_xmit_SLIP6", "_xmit_CSLIP6", "_xmit_RSRVD",
435 	"_xmit_ADAPT", "_xmit_ROSE", "_xmit_X25", "_xmit_HWX25",
436 	"_xmit_PPP", "_xmit_CISCO", "_xmit_LAPB", "_xmit_DDCMP",
437 	"_xmit_RAWHDLC", "_xmit_TUNNEL", "_xmit_TUNNEL6", "_xmit_FRAD",
438 	"_xmit_SKIP", "_xmit_LOOPBACK", "_xmit_LOCALTLK", "_xmit_FDDI",
439 	"_xmit_BIF", "_xmit_SIT", "_xmit_IPDDP", "_xmit_IPGRE",
440 	"_xmit_PIMREG", "_xmit_HIPPI", "_xmit_ASH", "_xmit_ECONET",
441 	"_xmit_IRDA", "_xmit_FCPP", "_xmit_FCAL", "_xmit_FCPL",
442 	"_xmit_FCFABRIC", "_xmit_IEEE80211", "_xmit_IEEE80211_PRISM",
443 	"_xmit_IEEE80211_RADIOTAP", "_xmit_PHONET", "_xmit_PHONET_PIPE",
444 	"_xmit_IEEE802154", "_xmit_VOID", "_xmit_NONE"};
445 
446 static struct lock_class_key netdev_xmit_lock_key[ARRAY_SIZE(netdev_lock_type)];
447 static struct lock_class_key netdev_addr_lock_key[ARRAY_SIZE(netdev_lock_type)];
448 
449 static inline unsigned short netdev_lock_pos(unsigned short dev_type)
450 {
451 	int i;
452 
453 	for (i = 0; i < ARRAY_SIZE(netdev_lock_type); i++)
454 		if (netdev_lock_type[i] == dev_type)
455 			return i;
456 	/* the last key is used by default */
457 	return ARRAY_SIZE(netdev_lock_type) - 1;
458 }
459 
460 static inline void netdev_set_xmit_lockdep_class(spinlock_t *lock,
461 						 unsigned short dev_type)
462 {
463 	int i;
464 
465 	i = netdev_lock_pos(dev_type);
466 	lockdep_set_class_and_name(lock, &netdev_xmit_lock_key[i],
467 				   netdev_lock_name[i]);
468 }
469 
470 static inline void netdev_set_addr_lockdep_class(struct net_device *dev)
471 {
472 	int i;
473 
474 	i = netdev_lock_pos(dev->type);
475 	lockdep_set_class_and_name(&dev->addr_list_lock,
476 				   &netdev_addr_lock_key[i],
477 				   netdev_lock_name[i]);
478 }
479 #else
480 static inline void netdev_set_xmit_lockdep_class(spinlock_t *lock,
481 						 unsigned short dev_type)
482 {
483 }
484 
485 static inline void netdev_set_addr_lockdep_class(struct net_device *dev)
486 {
487 }
488 #endif
489 
490 /*******************************************************************************
491  *
492  *		Protocol management and registration routines
493  *
494  *******************************************************************************/
495 
496 
497 /*
498  *	Add a protocol ID to the list. Now that the input handler is
499  *	smarter we can dispense with all the messy stuff that used to be
500  *	here.
501  *
502  *	BEWARE!!! Protocol handlers, mangling input packets,
503  *	MUST BE last in hash buckets and checking protocol handlers
504  *	MUST start from promiscuous ptype_all chain in net_bh.
505  *	It is true now, do not change it.
506  *	Explanation follows: if protocol handler, mangling packet, will
507  *	be the first on list, it is not able to sense, that packet
508  *	is cloned and should be copied-on-write, so that it will
509  *	change it and subsequent readers will get broken packet.
510  *							--ANK (980803)
511  */
512 
513 static inline struct list_head *ptype_head(const struct packet_type *pt)
514 {
515 	if (pt->type == htons(ETH_P_ALL))
516 		return pt->dev ? &pt->dev->ptype_all : &ptype_all;
517 	else
518 		return pt->dev ? &pt->dev->ptype_specific :
519 				 &ptype_base[ntohs(pt->type) & PTYPE_HASH_MASK];
520 }
521 
522 /**
523  *	dev_add_pack - add packet handler
524  *	@pt: packet type declaration
525  *
526  *	Add a protocol handler to the networking stack. The passed &packet_type
527  *	is linked into kernel lists and may not be freed until it has been
528  *	removed from the kernel lists.
529  *
530  *	This call does not sleep therefore it can not
531  *	guarantee all CPU's that are in middle of receiving packets
532  *	will see the new packet type (until the next received packet).
533  */
534 
535 void dev_add_pack(struct packet_type *pt)
536 {
537 	struct list_head *head = ptype_head(pt);
538 
539 	spin_lock(&ptype_lock);
540 	list_add_rcu(&pt->list, head);
541 	spin_unlock(&ptype_lock);
542 }
543 EXPORT_SYMBOL(dev_add_pack);
544 
545 /**
546  *	__dev_remove_pack	 - remove packet handler
547  *	@pt: packet type declaration
548  *
549  *	Remove a protocol handler that was previously added to the kernel
550  *	protocol handlers by dev_add_pack(). The passed &packet_type is removed
551  *	from the kernel lists and can be freed or reused once this function
552  *	returns.
553  *
554  *      The packet type might still be in use by receivers
555  *	and must not be freed until after all the CPU's have gone
556  *	through a quiescent state.
557  */
558 void __dev_remove_pack(struct packet_type *pt)
559 {
560 	struct list_head *head = ptype_head(pt);
561 	struct packet_type *pt1;
562 
563 	spin_lock(&ptype_lock);
564 
565 	list_for_each_entry(pt1, head, list) {
566 		if (pt == pt1) {
567 			list_del_rcu(&pt->list);
568 			goto out;
569 		}
570 	}
571 
572 	pr_warn("dev_remove_pack: %p not found\n", pt);
573 out:
574 	spin_unlock(&ptype_lock);
575 }
576 EXPORT_SYMBOL(__dev_remove_pack);
577 
578 /**
579  *	dev_remove_pack	 - remove packet handler
580  *	@pt: packet type declaration
581  *
582  *	Remove a protocol handler that was previously added to the kernel
583  *	protocol handlers by dev_add_pack(). The passed &packet_type is removed
584  *	from the kernel lists and can be freed or reused once this function
585  *	returns.
586  *
587  *	This call sleeps to guarantee that no CPU is looking at the packet
588  *	type after return.
589  */
590 void dev_remove_pack(struct packet_type *pt)
591 {
592 	__dev_remove_pack(pt);
593 
594 	synchronize_net();
595 }
596 EXPORT_SYMBOL(dev_remove_pack);
597 
598 
599 /**
600  *	dev_add_offload - register offload handlers
601  *	@po: protocol offload declaration
602  *
603  *	Add protocol offload handlers to the networking stack. The passed
604  *	&proto_offload is linked into kernel lists and may not be freed until
605  *	it has been removed from the kernel lists.
606  *
607  *	This call does not sleep therefore it can not
608  *	guarantee all CPU's that are in middle of receiving packets
609  *	will see the new offload handlers (until the next received packet).
610  */
611 void dev_add_offload(struct packet_offload *po)
612 {
613 	struct packet_offload *elem;
614 
615 	spin_lock(&offload_lock);
616 	list_for_each_entry(elem, &offload_base, list) {
617 		if (po->priority < elem->priority)
618 			break;
619 	}
620 	list_add_rcu(&po->list, elem->list.prev);
621 	spin_unlock(&offload_lock);
622 }
623 EXPORT_SYMBOL(dev_add_offload);
624 
625 /**
626  *	__dev_remove_offload	 - remove offload handler
627  *	@po: packet offload declaration
628  *
629  *	Remove a protocol offload handler that was previously added to the
630  *	kernel offload handlers by dev_add_offload(). The passed &offload_type
631  *	is removed from the kernel lists and can be freed or reused once this
632  *	function returns.
633  *
634  *      The packet type might still be in use by receivers
635  *	and must not be freed until after all the CPU's have gone
636  *	through a quiescent state.
637  */
638 static void __dev_remove_offload(struct packet_offload *po)
639 {
640 	struct list_head *head = &offload_base;
641 	struct packet_offload *po1;
642 
643 	spin_lock(&offload_lock);
644 
645 	list_for_each_entry(po1, head, list) {
646 		if (po == po1) {
647 			list_del_rcu(&po->list);
648 			goto out;
649 		}
650 	}
651 
652 	pr_warn("dev_remove_offload: %p not found\n", po);
653 out:
654 	spin_unlock(&offload_lock);
655 }
656 
657 /**
658  *	dev_remove_offload	 - remove packet offload handler
659  *	@po: packet offload declaration
660  *
661  *	Remove a packet offload handler that was previously added to the kernel
662  *	offload handlers by dev_add_offload(). The passed &offload_type is
663  *	removed from the kernel lists and can be freed or reused once this
664  *	function returns.
665  *
666  *	This call sleeps to guarantee that no CPU is looking at the packet
667  *	type after return.
668  */
669 void dev_remove_offload(struct packet_offload *po)
670 {
671 	__dev_remove_offload(po);
672 
673 	synchronize_net();
674 }
675 EXPORT_SYMBOL(dev_remove_offload);
676 
677 /******************************************************************************
678  *
679  *		      Device Boot-time Settings Routines
680  *
681  ******************************************************************************/
682 
683 /* Boot time configuration table */
684 static struct netdev_boot_setup dev_boot_setup[NETDEV_BOOT_SETUP_MAX];
685 
686 /**
687  *	netdev_boot_setup_add	- add new setup entry
688  *	@name: name of the device
689  *	@map: configured settings for the device
690  *
691  *	Adds new setup entry to the dev_boot_setup list.  The function
692  *	returns 0 on error and 1 on success.  This is a generic routine to
693  *	all netdevices.
694  */
695 static int netdev_boot_setup_add(char *name, struct ifmap *map)
696 {
697 	struct netdev_boot_setup *s;
698 	int i;
699 
700 	s = dev_boot_setup;
701 	for (i = 0; i < NETDEV_BOOT_SETUP_MAX; i++) {
702 		if (s[i].name[0] == '\0' || s[i].name[0] == ' ') {
703 			memset(s[i].name, 0, sizeof(s[i].name));
704 			strlcpy(s[i].name, name, IFNAMSIZ);
705 			memcpy(&s[i].map, map, sizeof(s[i].map));
706 			break;
707 		}
708 	}
709 
710 	return i >= NETDEV_BOOT_SETUP_MAX ? 0 : 1;
711 }
712 
713 /**
714  * netdev_boot_setup_check	- check boot time settings
715  * @dev: the netdevice
716  *
717  * Check boot time settings for the device.
718  * The found settings are set for the device to be used
719  * later in the device probing.
720  * Returns 0 if no settings found, 1 if they are.
721  */
722 int netdev_boot_setup_check(struct net_device *dev)
723 {
724 	struct netdev_boot_setup *s = dev_boot_setup;
725 	int i;
726 
727 	for (i = 0; i < NETDEV_BOOT_SETUP_MAX; i++) {
728 		if (s[i].name[0] != '\0' && s[i].name[0] != ' ' &&
729 		    !strcmp(dev->name, s[i].name)) {
730 			dev->irq = s[i].map.irq;
731 			dev->base_addr = s[i].map.base_addr;
732 			dev->mem_start = s[i].map.mem_start;
733 			dev->mem_end = s[i].map.mem_end;
734 			return 1;
735 		}
736 	}
737 	return 0;
738 }
739 EXPORT_SYMBOL(netdev_boot_setup_check);
740 
741 
742 /**
743  * netdev_boot_base	- get address from boot time settings
744  * @prefix: prefix for network device
745  * @unit: id for network device
746  *
747  * Check boot time settings for the base address of device.
748  * The found settings are set for the device to be used
749  * later in the device probing.
750  * Returns 0 if no settings found.
751  */
752 unsigned long netdev_boot_base(const char *prefix, int unit)
753 {
754 	const struct netdev_boot_setup *s = dev_boot_setup;
755 	char name[IFNAMSIZ];
756 	int i;
757 
758 	sprintf(name, "%s%d", prefix, unit);
759 
760 	/*
761 	 * If device already registered then return base of 1
762 	 * to indicate not to probe for this interface
763 	 */
764 	if (__dev_get_by_name(&init_net, name))
765 		return 1;
766 
767 	for (i = 0; i < NETDEV_BOOT_SETUP_MAX; i++)
768 		if (!strcmp(name, s[i].name))
769 			return s[i].map.base_addr;
770 	return 0;
771 }
772 
773 /*
774  * Saves at boot time configured settings for any netdevice.
775  */
776 int __init netdev_boot_setup(char *str)
777 {
778 	int ints[5];
779 	struct ifmap map;
780 
781 	str = get_options(str, ARRAY_SIZE(ints), ints);
782 	if (!str || !*str)
783 		return 0;
784 
785 	/* Save settings */
786 	memset(&map, 0, sizeof(map));
787 	if (ints[0] > 0)
788 		map.irq = ints[1];
789 	if (ints[0] > 1)
790 		map.base_addr = ints[2];
791 	if (ints[0] > 2)
792 		map.mem_start = ints[3];
793 	if (ints[0] > 3)
794 		map.mem_end = ints[4];
795 
796 	/* Add new entry to the list */
797 	return netdev_boot_setup_add(str, &map);
798 }
799 
800 __setup("netdev=", netdev_boot_setup);
801 
802 /*******************************************************************************
803  *
804  *			    Device Interface Subroutines
805  *
806  *******************************************************************************/
807 
808 /**
809  *	dev_get_iflink	- get 'iflink' value of a interface
810  *	@dev: targeted interface
811  *
812  *	Indicates the ifindex the interface is linked to.
813  *	Physical interfaces have the same 'ifindex' and 'iflink' values.
814  */
815 
816 int dev_get_iflink(const struct net_device *dev)
817 {
818 	if (dev->netdev_ops && dev->netdev_ops->ndo_get_iflink)
819 		return dev->netdev_ops->ndo_get_iflink(dev);
820 
821 	return dev->ifindex;
822 }
823 EXPORT_SYMBOL(dev_get_iflink);
824 
825 /**
826  *	dev_fill_metadata_dst - Retrieve tunnel egress information.
827  *	@dev: targeted interface
828  *	@skb: The packet.
829  *
830  *	For better visibility of tunnel traffic OVS needs to retrieve
831  *	egress tunnel information for a packet. Following API allows
832  *	user to get this info.
833  */
834 int dev_fill_metadata_dst(struct net_device *dev, struct sk_buff *skb)
835 {
836 	struct ip_tunnel_info *info;
837 
838 	if (!dev->netdev_ops  || !dev->netdev_ops->ndo_fill_metadata_dst)
839 		return -EINVAL;
840 
841 	info = skb_tunnel_info_unclone(skb);
842 	if (!info)
843 		return -ENOMEM;
844 	if (unlikely(!(info->mode & IP_TUNNEL_INFO_TX)))
845 		return -EINVAL;
846 
847 	return dev->netdev_ops->ndo_fill_metadata_dst(dev, skb);
848 }
849 EXPORT_SYMBOL_GPL(dev_fill_metadata_dst);
850 
851 /**
852  *	__dev_get_by_name	- find a device by its name
853  *	@net: the applicable net namespace
854  *	@name: name to find
855  *
856  *	Find an interface by name. Must be called under RTNL semaphore
857  *	or @dev_base_lock. If the name is found a pointer to the device
858  *	is returned. If the name is not found then %NULL is returned. The
859  *	reference counters are not incremented so the caller must be
860  *	careful with locks.
861  */
862 
863 struct net_device *__dev_get_by_name(struct net *net, const char *name)
864 {
865 	struct netdev_name_node *node_name;
866 
867 	node_name = netdev_name_node_lookup(net, name);
868 	return node_name ? node_name->dev : NULL;
869 }
870 EXPORT_SYMBOL(__dev_get_by_name);
871 
872 /**
873  * dev_get_by_name_rcu	- find a device by its name
874  * @net: the applicable net namespace
875  * @name: name to find
876  *
877  * Find an interface by name.
878  * If the name is found a pointer to the device is returned.
879  * If the name is not found then %NULL is returned.
880  * The reference counters are not incremented so the caller must be
881  * careful with locks. The caller must hold RCU lock.
882  */
883 
884 struct net_device *dev_get_by_name_rcu(struct net *net, const char *name)
885 {
886 	struct netdev_name_node *node_name;
887 
888 	node_name = netdev_name_node_lookup_rcu(net, name);
889 	return node_name ? node_name->dev : NULL;
890 }
891 EXPORT_SYMBOL(dev_get_by_name_rcu);
892 
893 /**
894  *	dev_get_by_name		- find a device by its name
895  *	@net: the applicable net namespace
896  *	@name: name to find
897  *
898  *	Find an interface by name. This can be called from any
899  *	context and does its own locking. The returned handle has
900  *	the usage count incremented and the caller must use dev_put() to
901  *	release it when it is no longer needed. %NULL is returned if no
902  *	matching device is found.
903  */
904 
905 struct net_device *dev_get_by_name(struct net *net, const char *name)
906 {
907 	struct net_device *dev;
908 
909 	rcu_read_lock();
910 	dev = dev_get_by_name_rcu(net, name);
911 	if (dev)
912 		dev_hold(dev);
913 	rcu_read_unlock();
914 	return dev;
915 }
916 EXPORT_SYMBOL(dev_get_by_name);
917 
918 /**
919  *	__dev_get_by_index - find a device by its ifindex
920  *	@net: the applicable net namespace
921  *	@ifindex: index of device
922  *
923  *	Search for an interface by index. Returns %NULL if the device
924  *	is not found or a pointer to the device. The device has not
925  *	had its reference counter increased so the caller must be careful
926  *	about locking. The caller must hold either the RTNL semaphore
927  *	or @dev_base_lock.
928  */
929 
930 struct net_device *__dev_get_by_index(struct net *net, int ifindex)
931 {
932 	struct net_device *dev;
933 	struct hlist_head *head = dev_index_hash(net, ifindex);
934 
935 	hlist_for_each_entry(dev, head, index_hlist)
936 		if (dev->ifindex == ifindex)
937 			return dev;
938 
939 	return NULL;
940 }
941 EXPORT_SYMBOL(__dev_get_by_index);
942 
943 /**
944  *	dev_get_by_index_rcu - find a device by its ifindex
945  *	@net: the applicable net namespace
946  *	@ifindex: index of device
947  *
948  *	Search for an interface by index. Returns %NULL if the device
949  *	is not found or a pointer to the device. The device has not
950  *	had its reference counter increased so the caller must be careful
951  *	about locking. The caller must hold RCU lock.
952  */
953 
954 struct net_device *dev_get_by_index_rcu(struct net *net, int ifindex)
955 {
956 	struct net_device *dev;
957 	struct hlist_head *head = dev_index_hash(net, ifindex);
958 
959 	hlist_for_each_entry_rcu(dev, head, index_hlist)
960 		if (dev->ifindex == ifindex)
961 			return dev;
962 
963 	return NULL;
964 }
965 EXPORT_SYMBOL(dev_get_by_index_rcu);
966 
967 
968 /**
969  *	dev_get_by_index - find a device by its ifindex
970  *	@net: the applicable net namespace
971  *	@ifindex: index of device
972  *
973  *	Search for an interface by index. Returns NULL if the device
974  *	is not found or a pointer to the device. The device returned has
975  *	had a reference added and the pointer is safe until the user calls
976  *	dev_put to indicate they have finished with it.
977  */
978 
979 struct net_device *dev_get_by_index(struct net *net, int ifindex)
980 {
981 	struct net_device *dev;
982 
983 	rcu_read_lock();
984 	dev = dev_get_by_index_rcu(net, ifindex);
985 	if (dev)
986 		dev_hold(dev);
987 	rcu_read_unlock();
988 	return dev;
989 }
990 EXPORT_SYMBOL(dev_get_by_index);
991 
992 /**
993  *	dev_get_by_napi_id - find a device by napi_id
994  *	@napi_id: ID of the NAPI struct
995  *
996  *	Search for an interface by NAPI ID. Returns %NULL if the device
997  *	is not found or a pointer to the device. The device has not had
998  *	its reference counter increased so the caller must be careful
999  *	about locking. The caller must hold RCU lock.
1000  */
1001 
1002 struct net_device *dev_get_by_napi_id(unsigned int napi_id)
1003 {
1004 	struct napi_struct *napi;
1005 
1006 	WARN_ON_ONCE(!rcu_read_lock_held());
1007 
1008 	if (napi_id < MIN_NAPI_ID)
1009 		return NULL;
1010 
1011 	napi = napi_by_id(napi_id);
1012 
1013 	return napi ? napi->dev : NULL;
1014 }
1015 EXPORT_SYMBOL(dev_get_by_napi_id);
1016 
1017 /**
1018  *	netdev_get_name - get a netdevice name, knowing its ifindex.
1019  *	@net: network namespace
1020  *	@name: a pointer to the buffer where the name will be stored.
1021  *	@ifindex: the ifindex of the interface to get the name from.
1022  */
1023 int netdev_get_name(struct net *net, char *name, int ifindex)
1024 {
1025 	struct net_device *dev;
1026 	int ret;
1027 
1028 	down_read(&devnet_rename_sem);
1029 	rcu_read_lock();
1030 
1031 	dev = dev_get_by_index_rcu(net, ifindex);
1032 	if (!dev) {
1033 		ret = -ENODEV;
1034 		goto out;
1035 	}
1036 
1037 	strcpy(name, dev->name);
1038 
1039 	ret = 0;
1040 out:
1041 	rcu_read_unlock();
1042 	up_read(&devnet_rename_sem);
1043 	return ret;
1044 }
1045 
1046 /**
1047  *	dev_getbyhwaddr_rcu - find a device by its hardware address
1048  *	@net: the applicable net namespace
1049  *	@type: media type of device
1050  *	@ha: hardware address
1051  *
1052  *	Search for an interface by MAC address. Returns NULL if the device
1053  *	is not found or a pointer to the device.
1054  *	The caller must hold RCU or RTNL.
1055  *	The returned device has not had its ref count increased
1056  *	and the caller must therefore be careful about locking
1057  *
1058  */
1059 
1060 struct net_device *dev_getbyhwaddr_rcu(struct net *net, unsigned short type,
1061 				       const char *ha)
1062 {
1063 	struct net_device *dev;
1064 
1065 	for_each_netdev_rcu(net, dev)
1066 		if (dev->type == type &&
1067 		    !memcmp(dev->dev_addr, ha, dev->addr_len))
1068 			return dev;
1069 
1070 	return NULL;
1071 }
1072 EXPORT_SYMBOL(dev_getbyhwaddr_rcu);
1073 
1074 struct net_device *dev_getfirstbyhwtype(struct net *net, unsigned short type)
1075 {
1076 	struct net_device *dev, *ret = NULL;
1077 
1078 	rcu_read_lock();
1079 	for_each_netdev_rcu(net, dev)
1080 		if (dev->type == type) {
1081 			dev_hold(dev);
1082 			ret = dev;
1083 			break;
1084 		}
1085 	rcu_read_unlock();
1086 	return ret;
1087 }
1088 EXPORT_SYMBOL(dev_getfirstbyhwtype);
1089 
1090 /**
1091  *	__dev_get_by_flags - find any device with given flags
1092  *	@net: the applicable net namespace
1093  *	@if_flags: IFF_* values
1094  *	@mask: bitmask of bits in if_flags to check
1095  *
1096  *	Search for any interface with the given flags. Returns NULL if a device
1097  *	is not found or a pointer to the device. Must be called inside
1098  *	rtnl_lock(), and result refcount is unchanged.
1099  */
1100 
1101 struct net_device *__dev_get_by_flags(struct net *net, unsigned short if_flags,
1102 				      unsigned short mask)
1103 {
1104 	struct net_device *dev, *ret;
1105 
1106 	ASSERT_RTNL();
1107 
1108 	ret = NULL;
1109 	for_each_netdev(net, dev) {
1110 		if (((dev->flags ^ if_flags) & mask) == 0) {
1111 			ret = dev;
1112 			break;
1113 		}
1114 	}
1115 	return ret;
1116 }
1117 EXPORT_SYMBOL(__dev_get_by_flags);
1118 
1119 /**
1120  *	dev_valid_name - check if name is okay for network device
1121  *	@name: name string
1122  *
1123  *	Network device names need to be valid file names to
1124  *	allow sysfs to work.  We also disallow any kind of
1125  *	whitespace.
1126  */
1127 bool dev_valid_name(const char *name)
1128 {
1129 	if (*name == '\0')
1130 		return false;
1131 	if (strnlen(name, IFNAMSIZ) == IFNAMSIZ)
1132 		return false;
1133 	if (!strcmp(name, ".") || !strcmp(name, ".."))
1134 		return false;
1135 
1136 	while (*name) {
1137 		if (*name == '/' || *name == ':' || isspace(*name))
1138 			return false;
1139 		name++;
1140 	}
1141 	return true;
1142 }
1143 EXPORT_SYMBOL(dev_valid_name);
1144 
1145 /**
1146  *	__dev_alloc_name - allocate a name for a device
1147  *	@net: network namespace to allocate the device name in
1148  *	@name: name format string
1149  *	@buf:  scratch buffer and result name string
1150  *
1151  *	Passed a format string - eg "lt%d" it will try and find a suitable
1152  *	id. It scans list of devices to build up a free map, then chooses
1153  *	the first empty slot. The caller must hold the dev_base or rtnl lock
1154  *	while allocating the name and adding the device in order to avoid
1155  *	duplicates.
1156  *	Limited to bits_per_byte * page size devices (ie 32K on most platforms).
1157  *	Returns the number of the unit assigned or a negative errno code.
1158  */
1159 
1160 static int __dev_alloc_name(struct net *net, const char *name, char *buf)
1161 {
1162 	int i = 0;
1163 	const char *p;
1164 	const int max_netdevices = 8*PAGE_SIZE;
1165 	unsigned long *inuse;
1166 	struct net_device *d;
1167 
1168 	if (!dev_valid_name(name))
1169 		return -EINVAL;
1170 
1171 	p = strchr(name, '%');
1172 	if (p) {
1173 		/*
1174 		 * Verify the string as this thing may have come from
1175 		 * the user.  There must be either one "%d" and no other "%"
1176 		 * characters.
1177 		 */
1178 		if (p[1] != 'd' || strchr(p + 2, '%'))
1179 			return -EINVAL;
1180 
1181 		/* Use one page as a bit array of possible slots */
1182 		inuse = (unsigned long *) get_zeroed_page(GFP_ATOMIC);
1183 		if (!inuse)
1184 			return -ENOMEM;
1185 
1186 		for_each_netdev(net, d) {
1187 			struct netdev_name_node *name_node;
1188 			list_for_each_entry(name_node, &d->name_node->list, list) {
1189 				if (!sscanf(name_node->name, name, &i))
1190 					continue;
1191 				if (i < 0 || i >= max_netdevices)
1192 					continue;
1193 
1194 				/*  avoid cases where sscanf is not exact inverse of printf */
1195 				snprintf(buf, IFNAMSIZ, name, i);
1196 				if (!strncmp(buf, name_node->name, IFNAMSIZ))
1197 					set_bit(i, inuse);
1198 			}
1199 			if (!sscanf(d->name, name, &i))
1200 				continue;
1201 			if (i < 0 || i >= max_netdevices)
1202 				continue;
1203 
1204 			/*  avoid cases where sscanf is not exact inverse of printf */
1205 			snprintf(buf, IFNAMSIZ, name, i);
1206 			if (!strncmp(buf, d->name, IFNAMSIZ))
1207 				set_bit(i, inuse);
1208 		}
1209 
1210 		i = find_first_zero_bit(inuse, max_netdevices);
1211 		free_page((unsigned long) inuse);
1212 	}
1213 
1214 	snprintf(buf, IFNAMSIZ, name, i);
1215 	if (!__dev_get_by_name(net, buf))
1216 		return i;
1217 
1218 	/* It is possible to run out of possible slots
1219 	 * when the name is long and there isn't enough space left
1220 	 * for the digits, or if all bits are used.
1221 	 */
1222 	return -ENFILE;
1223 }
1224 
1225 static int dev_alloc_name_ns(struct net *net,
1226 			     struct net_device *dev,
1227 			     const char *name)
1228 {
1229 	char buf[IFNAMSIZ];
1230 	int ret;
1231 
1232 	BUG_ON(!net);
1233 	ret = __dev_alloc_name(net, name, buf);
1234 	if (ret >= 0)
1235 		strlcpy(dev->name, buf, IFNAMSIZ);
1236 	return ret;
1237 }
1238 
1239 /**
1240  *	dev_alloc_name - allocate a name for a device
1241  *	@dev: device
1242  *	@name: name format string
1243  *
1244  *	Passed a format string - eg "lt%d" it will try and find a suitable
1245  *	id. It scans list of devices to build up a free map, then chooses
1246  *	the first empty slot. The caller must hold the dev_base or rtnl lock
1247  *	while allocating the name and adding the device in order to avoid
1248  *	duplicates.
1249  *	Limited to bits_per_byte * page size devices (ie 32K on most platforms).
1250  *	Returns the number of the unit assigned or a negative errno code.
1251  */
1252 
1253 int dev_alloc_name(struct net_device *dev, const char *name)
1254 {
1255 	return dev_alloc_name_ns(dev_net(dev), dev, name);
1256 }
1257 EXPORT_SYMBOL(dev_alloc_name);
1258 
1259 static int dev_get_valid_name(struct net *net, struct net_device *dev,
1260 			      const char *name)
1261 {
1262 	BUG_ON(!net);
1263 
1264 	if (!dev_valid_name(name))
1265 		return -EINVAL;
1266 
1267 	if (strchr(name, '%'))
1268 		return dev_alloc_name_ns(net, dev, name);
1269 	else if (__dev_get_by_name(net, name))
1270 		return -EEXIST;
1271 	else if (dev->name != name)
1272 		strlcpy(dev->name, name, IFNAMSIZ);
1273 
1274 	return 0;
1275 }
1276 
1277 /**
1278  *	dev_change_name - change name of a device
1279  *	@dev: device
1280  *	@newname: name (or format string) must be at least IFNAMSIZ
1281  *
1282  *	Change name of a device, can pass format strings "eth%d".
1283  *	for wildcarding.
1284  */
1285 int dev_change_name(struct net_device *dev, const char *newname)
1286 {
1287 	unsigned char old_assign_type;
1288 	char oldname[IFNAMSIZ];
1289 	int err = 0;
1290 	int ret;
1291 	struct net *net;
1292 
1293 	ASSERT_RTNL();
1294 	BUG_ON(!dev_net(dev));
1295 
1296 	net = dev_net(dev);
1297 
1298 	/* Some auto-enslaved devices e.g. failover slaves are
1299 	 * special, as userspace might rename the device after
1300 	 * the interface had been brought up and running since
1301 	 * the point kernel initiated auto-enslavement. Allow
1302 	 * live name change even when these slave devices are
1303 	 * up and running.
1304 	 *
1305 	 * Typically, users of these auto-enslaving devices
1306 	 * don't actually care about slave name change, as
1307 	 * they are supposed to operate on master interface
1308 	 * directly.
1309 	 */
1310 	if (dev->flags & IFF_UP &&
1311 	    likely(!(dev->priv_flags & IFF_LIVE_RENAME_OK)))
1312 		return -EBUSY;
1313 
1314 	down_write(&devnet_rename_sem);
1315 
1316 	if (strncmp(newname, dev->name, IFNAMSIZ) == 0) {
1317 		up_write(&devnet_rename_sem);
1318 		return 0;
1319 	}
1320 
1321 	memcpy(oldname, dev->name, IFNAMSIZ);
1322 
1323 	err = dev_get_valid_name(net, dev, newname);
1324 	if (err < 0) {
1325 		up_write(&devnet_rename_sem);
1326 		return err;
1327 	}
1328 
1329 	if (oldname[0] && !strchr(oldname, '%'))
1330 		netdev_info(dev, "renamed from %s\n", oldname);
1331 
1332 	old_assign_type = dev->name_assign_type;
1333 	dev->name_assign_type = NET_NAME_RENAMED;
1334 
1335 rollback:
1336 	ret = device_rename(&dev->dev, dev->name);
1337 	if (ret) {
1338 		memcpy(dev->name, oldname, IFNAMSIZ);
1339 		dev->name_assign_type = old_assign_type;
1340 		up_write(&devnet_rename_sem);
1341 		return ret;
1342 	}
1343 
1344 	up_write(&devnet_rename_sem);
1345 
1346 	netdev_adjacent_rename_links(dev, oldname);
1347 
1348 	write_lock_bh(&dev_base_lock);
1349 	netdev_name_node_del(dev->name_node);
1350 	write_unlock_bh(&dev_base_lock);
1351 
1352 	synchronize_rcu();
1353 
1354 	write_lock_bh(&dev_base_lock);
1355 	netdev_name_node_add(net, dev->name_node);
1356 	write_unlock_bh(&dev_base_lock);
1357 
1358 	ret = call_netdevice_notifiers(NETDEV_CHANGENAME, dev);
1359 	ret = notifier_to_errno(ret);
1360 
1361 	if (ret) {
1362 		/* err >= 0 after dev_alloc_name() or stores the first errno */
1363 		if (err >= 0) {
1364 			err = ret;
1365 			down_write(&devnet_rename_sem);
1366 			memcpy(dev->name, oldname, IFNAMSIZ);
1367 			memcpy(oldname, newname, IFNAMSIZ);
1368 			dev->name_assign_type = old_assign_type;
1369 			old_assign_type = NET_NAME_RENAMED;
1370 			goto rollback;
1371 		} else {
1372 			pr_err("%s: name change rollback failed: %d\n",
1373 			       dev->name, ret);
1374 		}
1375 	}
1376 
1377 	return err;
1378 }
1379 
1380 /**
1381  *	dev_set_alias - change ifalias of a device
1382  *	@dev: device
1383  *	@alias: name up to IFALIASZ
1384  *	@len: limit of bytes to copy from info
1385  *
1386  *	Set ifalias for a device,
1387  */
1388 int dev_set_alias(struct net_device *dev, const char *alias, size_t len)
1389 {
1390 	struct dev_ifalias *new_alias = NULL;
1391 
1392 	if (len >= IFALIASZ)
1393 		return -EINVAL;
1394 
1395 	if (len) {
1396 		new_alias = kmalloc(sizeof(*new_alias) + len + 1, GFP_KERNEL);
1397 		if (!new_alias)
1398 			return -ENOMEM;
1399 
1400 		memcpy(new_alias->ifalias, alias, len);
1401 		new_alias->ifalias[len] = 0;
1402 	}
1403 
1404 	mutex_lock(&ifalias_mutex);
1405 	new_alias = rcu_replace_pointer(dev->ifalias, new_alias,
1406 					mutex_is_locked(&ifalias_mutex));
1407 	mutex_unlock(&ifalias_mutex);
1408 
1409 	if (new_alias)
1410 		kfree_rcu(new_alias, rcuhead);
1411 
1412 	return len;
1413 }
1414 EXPORT_SYMBOL(dev_set_alias);
1415 
1416 /**
1417  *	dev_get_alias - get ifalias of a device
1418  *	@dev: device
1419  *	@name: buffer to store name of ifalias
1420  *	@len: size of buffer
1421  *
1422  *	get ifalias for a device.  Caller must make sure dev cannot go
1423  *	away,  e.g. rcu read lock or own a reference count to device.
1424  */
1425 int dev_get_alias(const struct net_device *dev, char *name, size_t len)
1426 {
1427 	const struct dev_ifalias *alias;
1428 	int ret = 0;
1429 
1430 	rcu_read_lock();
1431 	alias = rcu_dereference(dev->ifalias);
1432 	if (alias)
1433 		ret = snprintf(name, len, "%s", alias->ifalias);
1434 	rcu_read_unlock();
1435 
1436 	return ret;
1437 }
1438 
1439 /**
1440  *	netdev_features_change - device changes features
1441  *	@dev: device to cause notification
1442  *
1443  *	Called to indicate a device has changed features.
1444  */
1445 void netdev_features_change(struct net_device *dev)
1446 {
1447 	call_netdevice_notifiers(NETDEV_FEAT_CHANGE, dev);
1448 }
1449 EXPORT_SYMBOL(netdev_features_change);
1450 
1451 /**
1452  *	netdev_state_change - device changes state
1453  *	@dev: device to cause notification
1454  *
1455  *	Called to indicate a device has changed state. This function calls
1456  *	the notifier chains for netdev_chain and sends a NEWLINK message
1457  *	to the routing socket.
1458  */
1459 void netdev_state_change(struct net_device *dev)
1460 {
1461 	if (dev->flags & IFF_UP) {
1462 		struct netdev_notifier_change_info change_info = {
1463 			.info.dev = dev,
1464 		};
1465 
1466 		call_netdevice_notifiers_info(NETDEV_CHANGE,
1467 					      &change_info.info);
1468 		rtmsg_ifinfo(RTM_NEWLINK, dev, 0, GFP_KERNEL);
1469 	}
1470 }
1471 EXPORT_SYMBOL(netdev_state_change);
1472 
1473 /**
1474  * __netdev_notify_peers - notify network peers about existence of @dev,
1475  * to be called when rtnl lock is already held.
1476  * @dev: network device
1477  *
1478  * Generate traffic such that interested network peers are aware of
1479  * @dev, such as by generating a gratuitous ARP. This may be used when
1480  * a device wants to inform the rest of the network about some sort of
1481  * reconfiguration such as a failover event or virtual machine
1482  * migration.
1483  */
1484 void __netdev_notify_peers(struct net_device *dev)
1485 {
1486 	ASSERT_RTNL();
1487 	call_netdevice_notifiers(NETDEV_NOTIFY_PEERS, dev);
1488 	call_netdevice_notifiers(NETDEV_RESEND_IGMP, dev);
1489 }
1490 EXPORT_SYMBOL(__netdev_notify_peers);
1491 
1492 /**
1493  * netdev_notify_peers - notify network peers about existence of @dev
1494  * @dev: network device
1495  *
1496  * Generate traffic such that interested network peers are aware of
1497  * @dev, such as by generating a gratuitous ARP. This may be used when
1498  * a device wants to inform the rest of the network about some sort of
1499  * reconfiguration such as a failover event or virtual machine
1500  * migration.
1501  */
1502 void netdev_notify_peers(struct net_device *dev)
1503 {
1504 	rtnl_lock();
1505 	__netdev_notify_peers(dev);
1506 	rtnl_unlock();
1507 }
1508 EXPORT_SYMBOL(netdev_notify_peers);
1509 
1510 static int napi_threaded_poll(void *data);
1511 
1512 static int napi_kthread_create(struct napi_struct *n)
1513 {
1514 	int err = 0;
1515 
1516 	/* Create and wake up the kthread once to put it in
1517 	 * TASK_INTERRUPTIBLE mode to avoid the blocked task
1518 	 * warning and work with loadavg.
1519 	 */
1520 	n->thread = kthread_run(napi_threaded_poll, n, "napi/%s-%d",
1521 				n->dev->name, n->napi_id);
1522 	if (IS_ERR(n->thread)) {
1523 		err = PTR_ERR(n->thread);
1524 		pr_err("kthread_run failed with err %d\n", err);
1525 		n->thread = NULL;
1526 	}
1527 
1528 	return err;
1529 }
1530 
1531 static int __dev_open(struct net_device *dev, struct netlink_ext_ack *extack)
1532 {
1533 	const struct net_device_ops *ops = dev->netdev_ops;
1534 	int ret;
1535 
1536 	ASSERT_RTNL();
1537 
1538 	if (!netif_device_present(dev)) {
1539 		/* may be detached because parent is runtime-suspended */
1540 		if (dev->dev.parent)
1541 			pm_runtime_resume(dev->dev.parent);
1542 		if (!netif_device_present(dev))
1543 			return -ENODEV;
1544 	}
1545 
1546 	/* Block netpoll from trying to do any rx path servicing.
1547 	 * If we don't do this there is a chance ndo_poll_controller
1548 	 * or ndo_poll may be running while we open the device
1549 	 */
1550 	netpoll_poll_disable(dev);
1551 
1552 	ret = call_netdevice_notifiers_extack(NETDEV_PRE_UP, dev, extack);
1553 	ret = notifier_to_errno(ret);
1554 	if (ret)
1555 		return ret;
1556 
1557 	set_bit(__LINK_STATE_START, &dev->state);
1558 
1559 	if (ops->ndo_validate_addr)
1560 		ret = ops->ndo_validate_addr(dev);
1561 
1562 	if (!ret && ops->ndo_open)
1563 		ret = ops->ndo_open(dev);
1564 
1565 	netpoll_poll_enable(dev);
1566 
1567 	if (ret)
1568 		clear_bit(__LINK_STATE_START, &dev->state);
1569 	else {
1570 		dev->flags |= IFF_UP;
1571 		dev_set_rx_mode(dev);
1572 		dev_activate(dev);
1573 		add_device_randomness(dev->dev_addr, dev->addr_len);
1574 	}
1575 
1576 	return ret;
1577 }
1578 
1579 /**
1580  *	dev_open	- prepare an interface for use.
1581  *	@dev: device to open
1582  *	@extack: netlink extended ack
1583  *
1584  *	Takes a device from down to up state. The device's private open
1585  *	function is invoked and then the multicast lists are loaded. Finally
1586  *	the device is moved into the up state and a %NETDEV_UP message is
1587  *	sent to the netdev notifier chain.
1588  *
1589  *	Calling this function on an active interface is a nop. On a failure
1590  *	a negative errno code is returned.
1591  */
1592 int dev_open(struct net_device *dev, struct netlink_ext_ack *extack)
1593 {
1594 	int ret;
1595 
1596 	if (dev->flags & IFF_UP)
1597 		return 0;
1598 
1599 	ret = __dev_open(dev, extack);
1600 	if (ret < 0)
1601 		return ret;
1602 
1603 	rtmsg_ifinfo(RTM_NEWLINK, dev, IFF_UP|IFF_RUNNING, GFP_KERNEL);
1604 	call_netdevice_notifiers(NETDEV_UP, dev);
1605 
1606 	return ret;
1607 }
1608 EXPORT_SYMBOL(dev_open);
1609 
1610 static void __dev_close_many(struct list_head *head)
1611 {
1612 	struct net_device *dev;
1613 
1614 	ASSERT_RTNL();
1615 	might_sleep();
1616 
1617 	list_for_each_entry(dev, head, close_list) {
1618 		/* Temporarily disable netpoll until the interface is down */
1619 		netpoll_poll_disable(dev);
1620 
1621 		call_netdevice_notifiers(NETDEV_GOING_DOWN, dev);
1622 
1623 		clear_bit(__LINK_STATE_START, &dev->state);
1624 
1625 		/* Synchronize to scheduled poll. We cannot touch poll list, it
1626 		 * can be even on different cpu. So just clear netif_running().
1627 		 *
1628 		 * dev->stop() will invoke napi_disable() on all of it's
1629 		 * napi_struct instances on this device.
1630 		 */
1631 		smp_mb__after_atomic(); /* Commit netif_running(). */
1632 	}
1633 
1634 	dev_deactivate_many(head);
1635 
1636 	list_for_each_entry(dev, head, close_list) {
1637 		const struct net_device_ops *ops = dev->netdev_ops;
1638 
1639 		/*
1640 		 *	Call the device specific close. This cannot fail.
1641 		 *	Only if device is UP
1642 		 *
1643 		 *	We allow it to be called even after a DETACH hot-plug
1644 		 *	event.
1645 		 */
1646 		if (ops->ndo_stop)
1647 			ops->ndo_stop(dev);
1648 
1649 		dev->flags &= ~IFF_UP;
1650 		netpoll_poll_enable(dev);
1651 	}
1652 }
1653 
1654 static void __dev_close(struct net_device *dev)
1655 {
1656 	LIST_HEAD(single);
1657 
1658 	list_add(&dev->close_list, &single);
1659 	__dev_close_many(&single);
1660 	list_del(&single);
1661 }
1662 
1663 void dev_close_many(struct list_head *head, bool unlink)
1664 {
1665 	struct net_device *dev, *tmp;
1666 
1667 	/* Remove the devices that don't need to be closed */
1668 	list_for_each_entry_safe(dev, tmp, head, close_list)
1669 		if (!(dev->flags & IFF_UP))
1670 			list_del_init(&dev->close_list);
1671 
1672 	__dev_close_many(head);
1673 
1674 	list_for_each_entry_safe(dev, tmp, head, close_list) {
1675 		rtmsg_ifinfo(RTM_NEWLINK, dev, IFF_UP|IFF_RUNNING, GFP_KERNEL);
1676 		call_netdevice_notifiers(NETDEV_DOWN, dev);
1677 		if (unlink)
1678 			list_del_init(&dev->close_list);
1679 	}
1680 }
1681 EXPORT_SYMBOL(dev_close_many);
1682 
1683 /**
1684  *	dev_close - shutdown an interface.
1685  *	@dev: device to shutdown
1686  *
1687  *	This function moves an active device into down state. A
1688  *	%NETDEV_GOING_DOWN is sent to the netdev notifier chain. The device
1689  *	is then deactivated and finally a %NETDEV_DOWN is sent to the notifier
1690  *	chain.
1691  */
1692 void dev_close(struct net_device *dev)
1693 {
1694 	if (dev->flags & IFF_UP) {
1695 		LIST_HEAD(single);
1696 
1697 		list_add(&dev->close_list, &single);
1698 		dev_close_many(&single, true);
1699 		list_del(&single);
1700 	}
1701 }
1702 EXPORT_SYMBOL(dev_close);
1703 
1704 
1705 /**
1706  *	dev_disable_lro - disable Large Receive Offload on a device
1707  *	@dev: device
1708  *
1709  *	Disable Large Receive Offload (LRO) on a net device.  Must be
1710  *	called under RTNL.  This is needed if received packets may be
1711  *	forwarded to another interface.
1712  */
1713 void dev_disable_lro(struct net_device *dev)
1714 {
1715 	struct net_device *lower_dev;
1716 	struct list_head *iter;
1717 
1718 	dev->wanted_features &= ~NETIF_F_LRO;
1719 	netdev_update_features(dev);
1720 
1721 	if (unlikely(dev->features & NETIF_F_LRO))
1722 		netdev_WARN(dev, "failed to disable LRO!\n");
1723 
1724 	netdev_for_each_lower_dev(dev, lower_dev, iter)
1725 		dev_disable_lro(lower_dev);
1726 }
1727 EXPORT_SYMBOL(dev_disable_lro);
1728 
1729 /**
1730  *	dev_disable_gro_hw - disable HW Generic Receive Offload on a device
1731  *	@dev: device
1732  *
1733  *	Disable HW Generic Receive Offload (GRO_HW) on a net device.  Must be
1734  *	called under RTNL.  This is needed if Generic XDP is installed on
1735  *	the device.
1736  */
1737 static void dev_disable_gro_hw(struct net_device *dev)
1738 {
1739 	dev->wanted_features &= ~NETIF_F_GRO_HW;
1740 	netdev_update_features(dev);
1741 
1742 	if (unlikely(dev->features & NETIF_F_GRO_HW))
1743 		netdev_WARN(dev, "failed to disable GRO_HW!\n");
1744 }
1745 
1746 const char *netdev_cmd_to_name(enum netdev_cmd cmd)
1747 {
1748 #define N(val) 						\
1749 	case NETDEV_##val:				\
1750 		return "NETDEV_" __stringify(val);
1751 	switch (cmd) {
1752 	N(UP) N(DOWN) N(REBOOT) N(CHANGE) N(REGISTER) N(UNREGISTER)
1753 	N(CHANGEMTU) N(CHANGEADDR) N(GOING_DOWN) N(CHANGENAME) N(FEAT_CHANGE)
1754 	N(BONDING_FAILOVER) N(PRE_UP) N(PRE_TYPE_CHANGE) N(POST_TYPE_CHANGE)
1755 	N(POST_INIT) N(RELEASE) N(NOTIFY_PEERS) N(JOIN) N(CHANGEUPPER)
1756 	N(RESEND_IGMP) N(PRECHANGEMTU) N(CHANGEINFODATA) N(BONDING_INFO)
1757 	N(PRECHANGEUPPER) N(CHANGELOWERSTATE) N(UDP_TUNNEL_PUSH_INFO)
1758 	N(UDP_TUNNEL_DROP_INFO) N(CHANGE_TX_QUEUE_LEN)
1759 	N(CVLAN_FILTER_PUSH_INFO) N(CVLAN_FILTER_DROP_INFO)
1760 	N(SVLAN_FILTER_PUSH_INFO) N(SVLAN_FILTER_DROP_INFO)
1761 	N(PRE_CHANGEADDR)
1762 	}
1763 #undef N
1764 	return "UNKNOWN_NETDEV_EVENT";
1765 }
1766 EXPORT_SYMBOL_GPL(netdev_cmd_to_name);
1767 
1768 static int call_netdevice_notifier(struct notifier_block *nb, unsigned long val,
1769 				   struct net_device *dev)
1770 {
1771 	struct netdev_notifier_info info = {
1772 		.dev = dev,
1773 	};
1774 
1775 	return nb->notifier_call(nb, val, &info);
1776 }
1777 
1778 static int call_netdevice_register_notifiers(struct notifier_block *nb,
1779 					     struct net_device *dev)
1780 {
1781 	int err;
1782 
1783 	err = call_netdevice_notifier(nb, NETDEV_REGISTER, dev);
1784 	err = notifier_to_errno(err);
1785 	if (err)
1786 		return err;
1787 
1788 	if (!(dev->flags & IFF_UP))
1789 		return 0;
1790 
1791 	call_netdevice_notifier(nb, NETDEV_UP, dev);
1792 	return 0;
1793 }
1794 
1795 static void call_netdevice_unregister_notifiers(struct notifier_block *nb,
1796 						struct net_device *dev)
1797 {
1798 	if (dev->flags & IFF_UP) {
1799 		call_netdevice_notifier(nb, NETDEV_GOING_DOWN,
1800 					dev);
1801 		call_netdevice_notifier(nb, NETDEV_DOWN, dev);
1802 	}
1803 	call_netdevice_notifier(nb, NETDEV_UNREGISTER, dev);
1804 }
1805 
1806 static int call_netdevice_register_net_notifiers(struct notifier_block *nb,
1807 						 struct net *net)
1808 {
1809 	struct net_device *dev;
1810 	int err;
1811 
1812 	for_each_netdev(net, dev) {
1813 		err = call_netdevice_register_notifiers(nb, dev);
1814 		if (err)
1815 			goto rollback;
1816 	}
1817 	return 0;
1818 
1819 rollback:
1820 	for_each_netdev_continue_reverse(net, dev)
1821 		call_netdevice_unregister_notifiers(nb, dev);
1822 	return err;
1823 }
1824 
1825 static void call_netdevice_unregister_net_notifiers(struct notifier_block *nb,
1826 						    struct net *net)
1827 {
1828 	struct net_device *dev;
1829 
1830 	for_each_netdev(net, dev)
1831 		call_netdevice_unregister_notifiers(nb, dev);
1832 }
1833 
1834 static int dev_boot_phase = 1;
1835 
1836 /**
1837  * register_netdevice_notifier - register a network notifier block
1838  * @nb: notifier
1839  *
1840  * Register a notifier to be called when network device events occur.
1841  * The notifier passed is linked into the kernel structures and must
1842  * not be reused until it has been unregistered. A negative errno code
1843  * is returned on a failure.
1844  *
1845  * When registered all registration and up events are replayed
1846  * to the new notifier to allow device to have a race free
1847  * view of the network device list.
1848  */
1849 
1850 int register_netdevice_notifier(struct notifier_block *nb)
1851 {
1852 	struct net *net;
1853 	int err;
1854 
1855 	/* Close race with setup_net() and cleanup_net() */
1856 	down_write(&pernet_ops_rwsem);
1857 	rtnl_lock();
1858 	err = raw_notifier_chain_register(&netdev_chain, nb);
1859 	if (err)
1860 		goto unlock;
1861 	if (dev_boot_phase)
1862 		goto unlock;
1863 	for_each_net(net) {
1864 		err = call_netdevice_register_net_notifiers(nb, net);
1865 		if (err)
1866 			goto rollback;
1867 	}
1868 
1869 unlock:
1870 	rtnl_unlock();
1871 	up_write(&pernet_ops_rwsem);
1872 	return err;
1873 
1874 rollback:
1875 	for_each_net_continue_reverse(net)
1876 		call_netdevice_unregister_net_notifiers(nb, net);
1877 
1878 	raw_notifier_chain_unregister(&netdev_chain, nb);
1879 	goto unlock;
1880 }
1881 EXPORT_SYMBOL(register_netdevice_notifier);
1882 
1883 /**
1884  * unregister_netdevice_notifier - unregister a network notifier block
1885  * @nb: notifier
1886  *
1887  * Unregister a notifier previously registered by
1888  * register_netdevice_notifier(). The notifier is unlinked into the
1889  * kernel structures and may then be reused. A negative errno code
1890  * is returned on a failure.
1891  *
1892  * After unregistering unregister and down device events are synthesized
1893  * for all devices on the device list to the removed notifier to remove
1894  * the need for special case cleanup code.
1895  */
1896 
1897 int unregister_netdevice_notifier(struct notifier_block *nb)
1898 {
1899 	struct net *net;
1900 	int err;
1901 
1902 	/* Close race with setup_net() and cleanup_net() */
1903 	down_write(&pernet_ops_rwsem);
1904 	rtnl_lock();
1905 	err = raw_notifier_chain_unregister(&netdev_chain, nb);
1906 	if (err)
1907 		goto unlock;
1908 
1909 	for_each_net(net)
1910 		call_netdevice_unregister_net_notifiers(nb, net);
1911 
1912 unlock:
1913 	rtnl_unlock();
1914 	up_write(&pernet_ops_rwsem);
1915 	return err;
1916 }
1917 EXPORT_SYMBOL(unregister_netdevice_notifier);
1918 
1919 static int __register_netdevice_notifier_net(struct net *net,
1920 					     struct notifier_block *nb,
1921 					     bool ignore_call_fail)
1922 {
1923 	int err;
1924 
1925 	err = raw_notifier_chain_register(&net->netdev_chain, nb);
1926 	if (err)
1927 		return err;
1928 	if (dev_boot_phase)
1929 		return 0;
1930 
1931 	err = call_netdevice_register_net_notifiers(nb, net);
1932 	if (err && !ignore_call_fail)
1933 		goto chain_unregister;
1934 
1935 	return 0;
1936 
1937 chain_unregister:
1938 	raw_notifier_chain_unregister(&net->netdev_chain, nb);
1939 	return err;
1940 }
1941 
1942 static int __unregister_netdevice_notifier_net(struct net *net,
1943 					       struct notifier_block *nb)
1944 {
1945 	int err;
1946 
1947 	err = raw_notifier_chain_unregister(&net->netdev_chain, nb);
1948 	if (err)
1949 		return err;
1950 
1951 	call_netdevice_unregister_net_notifiers(nb, net);
1952 	return 0;
1953 }
1954 
1955 /**
1956  * register_netdevice_notifier_net - register a per-netns network notifier block
1957  * @net: network namespace
1958  * @nb: notifier
1959  *
1960  * Register a notifier to be called when network device events occur.
1961  * The notifier passed is linked into the kernel structures and must
1962  * not be reused until it has been unregistered. A negative errno code
1963  * is returned on a failure.
1964  *
1965  * When registered all registration and up events are replayed
1966  * to the new notifier to allow device to have a race free
1967  * view of the network device list.
1968  */
1969 
1970 int register_netdevice_notifier_net(struct net *net, struct notifier_block *nb)
1971 {
1972 	int err;
1973 
1974 	rtnl_lock();
1975 	err = __register_netdevice_notifier_net(net, nb, false);
1976 	rtnl_unlock();
1977 	return err;
1978 }
1979 EXPORT_SYMBOL(register_netdevice_notifier_net);
1980 
1981 /**
1982  * unregister_netdevice_notifier_net - unregister a per-netns
1983  *                                     network notifier block
1984  * @net: network namespace
1985  * @nb: notifier
1986  *
1987  * Unregister a notifier previously registered by
1988  * register_netdevice_notifier(). The notifier is unlinked into the
1989  * kernel structures and may then be reused. A negative errno code
1990  * is returned on a failure.
1991  *
1992  * After unregistering unregister and down device events are synthesized
1993  * for all devices on the device list to the removed notifier to remove
1994  * the need for special case cleanup code.
1995  */
1996 
1997 int unregister_netdevice_notifier_net(struct net *net,
1998 				      struct notifier_block *nb)
1999 {
2000 	int err;
2001 
2002 	rtnl_lock();
2003 	err = __unregister_netdevice_notifier_net(net, nb);
2004 	rtnl_unlock();
2005 	return err;
2006 }
2007 EXPORT_SYMBOL(unregister_netdevice_notifier_net);
2008 
2009 int register_netdevice_notifier_dev_net(struct net_device *dev,
2010 					struct notifier_block *nb,
2011 					struct netdev_net_notifier *nn)
2012 {
2013 	int err;
2014 
2015 	rtnl_lock();
2016 	err = __register_netdevice_notifier_net(dev_net(dev), nb, false);
2017 	if (!err) {
2018 		nn->nb = nb;
2019 		list_add(&nn->list, &dev->net_notifier_list);
2020 	}
2021 	rtnl_unlock();
2022 	return err;
2023 }
2024 EXPORT_SYMBOL(register_netdevice_notifier_dev_net);
2025 
2026 int unregister_netdevice_notifier_dev_net(struct net_device *dev,
2027 					  struct notifier_block *nb,
2028 					  struct netdev_net_notifier *nn)
2029 {
2030 	int err;
2031 
2032 	rtnl_lock();
2033 	list_del(&nn->list);
2034 	err = __unregister_netdevice_notifier_net(dev_net(dev), nb);
2035 	rtnl_unlock();
2036 	return err;
2037 }
2038 EXPORT_SYMBOL(unregister_netdevice_notifier_dev_net);
2039 
2040 static void move_netdevice_notifiers_dev_net(struct net_device *dev,
2041 					     struct net *net)
2042 {
2043 	struct netdev_net_notifier *nn;
2044 
2045 	list_for_each_entry(nn, &dev->net_notifier_list, list) {
2046 		__unregister_netdevice_notifier_net(dev_net(dev), nn->nb);
2047 		__register_netdevice_notifier_net(net, nn->nb, true);
2048 	}
2049 }
2050 
2051 /**
2052  *	call_netdevice_notifiers_info - call all network notifier blocks
2053  *	@val: value passed unmodified to notifier function
2054  *	@info: notifier information data
2055  *
2056  *	Call all network notifier blocks.  Parameters and return value
2057  *	are as for raw_notifier_call_chain().
2058  */
2059 
2060 static int call_netdevice_notifiers_info(unsigned long val,
2061 					 struct netdev_notifier_info *info)
2062 {
2063 	struct net *net = dev_net(info->dev);
2064 	int ret;
2065 
2066 	ASSERT_RTNL();
2067 
2068 	/* Run per-netns notifier block chain first, then run the global one.
2069 	 * Hopefully, one day, the global one is going to be removed after
2070 	 * all notifier block registrators get converted to be per-netns.
2071 	 */
2072 	ret = raw_notifier_call_chain(&net->netdev_chain, val, info);
2073 	if (ret & NOTIFY_STOP_MASK)
2074 		return ret;
2075 	return raw_notifier_call_chain(&netdev_chain, val, info);
2076 }
2077 
2078 static int call_netdevice_notifiers_extack(unsigned long val,
2079 					   struct net_device *dev,
2080 					   struct netlink_ext_ack *extack)
2081 {
2082 	struct netdev_notifier_info info = {
2083 		.dev = dev,
2084 		.extack = extack,
2085 	};
2086 
2087 	return call_netdevice_notifiers_info(val, &info);
2088 }
2089 
2090 /**
2091  *	call_netdevice_notifiers - call all network notifier blocks
2092  *      @val: value passed unmodified to notifier function
2093  *      @dev: net_device pointer passed unmodified to notifier function
2094  *
2095  *	Call all network notifier blocks.  Parameters and return value
2096  *	are as for raw_notifier_call_chain().
2097  */
2098 
2099 int call_netdevice_notifiers(unsigned long val, struct net_device *dev)
2100 {
2101 	return call_netdevice_notifiers_extack(val, dev, NULL);
2102 }
2103 EXPORT_SYMBOL(call_netdevice_notifiers);
2104 
2105 /**
2106  *	call_netdevice_notifiers_mtu - call all network notifier blocks
2107  *	@val: value passed unmodified to notifier function
2108  *	@dev: net_device pointer passed unmodified to notifier function
2109  *	@arg: additional u32 argument passed to the notifier function
2110  *
2111  *	Call all network notifier blocks.  Parameters and return value
2112  *	are as for raw_notifier_call_chain().
2113  */
2114 static int call_netdevice_notifiers_mtu(unsigned long val,
2115 					struct net_device *dev, u32 arg)
2116 {
2117 	struct netdev_notifier_info_ext info = {
2118 		.info.dev = dev,
2119 		.ext.mtu = arg,
2120 	};
2121 
2122 	BUILD_BUG_ON(offsetof(struct netdev_notifier_info_ext, info) != 0);
2123 
2124 	return call_netdevice_notifiers_info(val, &info.info);
2125 }
2126 
2127 #ifdef CONFIG_NET_INGRESS
2128 static DEFINE_STATIC_KEY_FALSE(ingress_needed_key);
2129 
2130 void net_inc_ingress_queue(void)
2131 {
2132 	static_branch_inc(&ingress_needed_key);
2133 }
2134 EXPORT_SYMBOL_GPL(net_inc_ingress_queue);
2135 
2136 void net_dec_ingress_queue(void)
2137 {
2138 	static_branch_dec(&ingress_needed_key);
2139 }
2140 EXPORT_SYMBOL_GPL(net_dec_ingress_queue);
2141 #endif
2142 
2143 #ifdef CONFIG_NET_EGRESS
2144 static DEFINE_STATIC_KEY_FALSE(egress_needed_key);
2145 
2146 void net_inc_egress_queue(void)
2147 {
2148 	static_branch_inc(&egress_needed_key);
2149 }
2150 EXPORT_SYMBOL_GPL(net_inc_egress_queue);
2151 
2152 void net_dec_egress_queue(void)
2153 {
2154 	static_branch_dec(&egress_needed_key);
2155 }
2156 EXPORT_SYMBOL_GPL(net_dec_egress_queue);
2157 #endif
2158 
2159 static DEFINE_STATIC_KEY_FALSE(netstamp_needed_key);
2160 #ifdef CONFIG_JUMP_LABEL
2161 static atomic_t netstamp_needed_deferred;
2162 static atomic_t netstamp_wanted;
2163 static void netstamp_clear(struct work_struct *work)
2164 {
2165 	int deferred = atomic_xchg(&netstamp_needed_deferred, 0);
2166 	int wanted;
2167 
2168 	wanted = atomic_add_return(deferred, &netstamp_wanted);
2169 	if (wanted > 0)
2170 		static_branch_enable(&netstamp_needed_key);
2171 	else
2172 		static_branch_disable(&netstamp_needed_key);
2173 }
2174 static DECLARE_WORK(netstamp_work, netstamp_clear);
2175 #endif
2176 
2177 void net_enable_timestamp(void)
2178 {
2179 #ifdef CONFIG_JUMP_LABEL
2180 	int wanted;
2181 
2182 	while (1) {
2183 		wanted = atomic_read(&netstamp_wanted);
2184 		if (wanted <= 0)
2185 			break;
2186 		if (atomic_cmpxchg(&netstamp_wanted, wanted, wanted + 1) == wanted)
2187 			return;
2188 	}
2189 	atomic_inc(&netstamp_needed_deferred);
2190 	schedule_work(&netstamp_work);
2191 #else
2192 	static_branch_inc(&netstamp_needed_key);
2193 #endif
2194 }
2195 EXPORT_SYMBOL(net_enable_timestamp);
2196 
2197 void net_disable_timestamp(void)
2198 {
2199 #ifdef CONFIG_JUMP_LABEL
2200 	int wanted;
2201 
2202 	while (1) {
2203 		wanted = atomic_read(&netstamp_wanted);
2204 		if (wanted <= 1)
2205 			break;
2206 		if (atomic_cmpxchg(&netstamp_wanted, wanted, wanted - 1) == wanted)
2207 			return;
2208 	}
2209 	atomic_dec(&netstamp_needed_deferred);
2210 	schedule_work(&netstamp_work);
2211 #else
2212 	static_branch_dec(&netstamp_needed_key);
2213 #endif
2214 }
2215 EXPORT_SYMBOL(net_disable_timestamp);
2216 
2217 static inline void net_timestamp_set(struct sk_buff *skb)
2218 {
2219 	skb->tstamp = 0;
2220 	if (static_branch_unlikely(&netstamp_needed_key))
2221 		__net_timestamp(skb);
2222 }
2223 
2224 #define net_timestamp_check(COND, SKB)				\
2225 	if (static_branch_unlikely(&netstamp_needed_key)) {	\
2226 		if ((COND) && !(SKB)->tstamp)			\
2227 			__net_timestamp(SKB);			\
2228 	}							\
2229 
2230 bool is_skb_forwardable(const struct net_device *dev, const struct sk_buff *skb)
2231 {
2232 	return __is_skb_forwardable(dev, skb, true);
2233 }
2234 EXPORT_SYMBOL_GPL(is_skb_forwardable);
2235 
2236 static int __dev_forward_skb2(struct net_device *dev, struct sk_buff *skb,
2237 			      bool check_mtu)
2238 {
2239 	int ret = ____dev_forward_skb(dev, skb, check_mtu);
2240 
2241 	if (likely(!ret)) {
2242 		skb->protocol = eth_type_trans(skb, dev);
2243 		skb_postpull_rcsum(skb, eth_hdr(skb), ETH_HLEN);
2244 	}
2245 
2246 	return ret;
2247 }
2248 
2249 int __dev_forward_skb(struct net_device *dev, struct sk_buff *skb)
2250 {
2251 	return __dev_forward_skb2(dev, skb, true);
2252 }
2253 EXPORT_SYMBOL_GPL(__dev_forward_skb);
2254 
2255 /**
2256  * dev_forward_skb - loopback an skb to another netif
2257  *
2258  * @dev: destination network device
2259  * @skb: buffer to forward
2260  *
2261  * return values:
2262  *	NET_RX_SUCCESS	(no congestion)
2263  *	NET_RX_DROP     (packet was dropped, but freed)
2264  *
2265  * dev_forward_skb can be used for injecting an skb from the
2266  * start_xmit function of one device into the receive queue
2267  * of another device.
2268  *
2269  * The receiving device may be in another namespace, so
2270  * we have to clear all information in the skb that could
2271  * impact namespace isolation.
2272  */
2273 int dev_forward_skb(struct net_device *dev, struct sk_buff *skb)
2274 {
2275 	return __dev_forward_skb(dev, skb) ?: netif_rx_internal(skb);
2276 }
2277 EXPORT_SYMBOL_GPL(dev_forward_skb);
2278 
2279 int dev_forward_skb_nomtu(struct net_device *dev, struct sk_buff *skb)
2280 {
2281 	return __dev_forward_skb2(dev, skb, false) ?: netif_rx_internal(skb);
2282 }
2283 
2284 static inline int deliver_skb(struct sk_buff *skb,
2285 			      struct packet_type *pt_prev,
2286 			      struct net_device *orig_dev)
2287 {
2288 	if (unlikely(skb_orphan_frags_rx(skb, GFP_ATOMIC)))
2289 		return -ENOMEM;
2290 	refcount_inc(&skb->users);
2291 	return pt_prev->func(skb, skb->dev, pt_prev, orig_dev);
2292 }
2293 
2294 static inline void deliver_ptype_list_skb(struct sk_buff *skb,
2295 					  struct packet_type **pt,
2296 					  struct net_device *orig_dev,
2297 					  __be16 type,
2298 					  struct list_head *ptype_list)
2299 {
2300 	struct packet_type *ptype, *pt_prev = *pt;
2301 
2302 	list_for_each_entry_rcu(ptype, ptype_list, list) {
2303 		if (ptype->type != type)
2304 			continue;
2305 		if (pt_prev)
2306 			deliver_skb(skb, pt_prev, orig_dev);
2307 		pt_prev = ptype;
2308 	}
2309 	*pt = pt_prev;
2310 }
2311 
2312 static inline bool skb_loop_sk(struct packet_type *ptype, struct sk_buff *skb)
2313 {
2314 	if (!ptype->af_packet_priv || !skb->sk)
2315 		return false;
2316 
2317 	if (ptype->id_match)
2318 		return ptype->id_match(ptype, skb->sk);
2319 	else if ((struct sock *)ptype->af_packet_priv == skb->sk)
2320 		return true;
2321 
2322 	return false;
2323 }
2324 
2325 /**
2326  * dev_nit_active - return true if any network interface taps are in use
2327  *
2328  * @dev: network device to check for the presence of taps
2329  */
2330 bool dev_nit_active(struct net_device *dev)
2331 {
2332 	return !list_empty(&ptype_all) || !list_empty(&dev->ptype_all);
2333 }
2334 EXPORT_SYMBOL_GPL(dev_nit_active);
2335 
2336 /*
2337  *	Support routine. Sends outgoing frames to any network
2338  *	taps currently in use.
2339  */
2340 
2341 void dev_queue_xmit_nit(struct sk_buff *skb, struct net_device *dev)
2342 {
2343 	struct packet_type *ptype;
2344 	struct sk_buff *skb2 = NULL;
2345 	struct packet_type *pt_prev = NULL;
2346 	struct list_head *ptype_list = &ptype_all;
2347 
2348 	rcu_read_lock();
2349 again:
2350 	list_for_each_entry_rcu(ptype, ptype_list, list) {
2351 		if (ptype->ignore_outgoing)
2352 			continue;
2353 
2354 		/* Never send packets back to the socket
2355 		 * they originated from - MvS (miquels@drinkel.ow.org)
2356 		 */
2357 		if (skb_loop_sk(ptype, skb))
2358 			continue;
2359 
2360 		if (pt_prev) {
2361 			deliver_skb(skb2, pt_prev, skb->dev);
2362 			pt_prev = ptype;
2363 			continue;
2364 		}
2365 
2366 		/* need to clone skb, done only once */
2367 		skb2 = skb_clone(skb, GFP_ATOMIC);
2368 		if (!skb2)
2369 			goto out_unlock;
2370 
2371 		net_timestamp_set(skb2);
2372 
2373 		/* skb->nh should be correctly
2374 		 * set by sender, so that the second statement is
2375 		 * just protection against buggy protocols.
2376 		 */
2377 		skb_reset_mac_header(skb2);
2378 
2379 		if (skb_network_header(skb2) < skb2->data ||
2380 		    skb_network_header(skb2) > skb_tail_pointer(skb2)) {
2381 			net_crit_ratelimited("protocol %04x is buggy, dev %s\n",
2382 					     ntohs(skb2->protocol),
2383 					     dev->name);
2384 			skb_reset_network_header(skb2);
2385 		}
2386 
2387 		skb2->transport_header = skb2->network_header;
2388 		skb2->pkt_type = PACKET_OUTGOING;
2389 		pt_prev = ptype;
2390 	}
2391 
2392 	if (ptype_list == &ptype_all) {
2393 		ptype_list = &dev->ptype_all;
2394 		goto again;
2395 	}
2396 out_unlock:
2397 	if (pt_prev) {
2398 		if (!skb_orphan_frags_rx(skb2, GFP_ATOMIC))
2399 			pt_prev->func(skb2, skb->dev, pt_prev, skb->dev);
2400 		else
2401 			kfree_skb(skb2);
2402 	}
2403 	rcu_read_unlock();
2404 }
2405 EXPORT_SYMBOL_GPL(dev_queue_xmit_nit);
2406 
2407 /**
2408  * netif_setup_tc - Handle tc mappings on real_num_tx_queues change
2409  * @dev: Network device
2410  * @txq: number of queues available
2411  *
2412  * If real_num_tx_queues is changed the tc mappings may no longer be
2413  * valid. To resolve this verify the tc mapping remains valid and if
2414  * not NULL the mapping. With no priorities mapping to this
2415  * offset/count pair it will no longer be used. In the worst case TC0
2416  * is invalid nothing can be done so disable priority mappings. If is
2417  * expected that drivers will fix this mapping if they can before
2418  * calling netif_set_real_num_tx_queues.
2419  */
2420 static void netif_setup_tc(struct net_device *dev, unsigned int txq)
2421 {
2422 	int i;
2423 	struct netdev_tc_txq *tc = &dev->tc_to_txq[0];
2424 
2425 	/* If TC0 is invalidated disable TC mapping */
2426 	if (tc->offset + tc->count > txq) {
2427 		pr_warn("Number of in use tx queues changed invalidating tc mappings. Priority traffic classification disabled!\n");
2428 		dev->num_tc = 0;
2429 		return;
2430 	}
2431 
2432 	/* Invalidated prio to tc mappings set to TC0 */
2433 	for (i = 1; i < TC_BITMASK + 1; i++) {
2434 		int q = netdev_get_prio_tc_map(dev, i);
2435 
2436 		tc = &dev->tc_to_txq[q];
2437 		if (tc->offset + tc->count > txq) {
2438 			pr_warn("Number of in use tx queues changed. Priority %i to tc mapping %i is no longer valid. Setting map to 0\n",
2439 				i, q);
2440 			netdev_set_prio_tc_map(dev, i, 0);
2441 		}
2442 	}
2443 }
2444 
2445 int netdev_txq_to_tc(struct net_device *dev, unsigned int txq)
2446 {
2447 	if (dev->num_tc) {
2448 		struct netdev_tc_txq *tc = &dev->tc_to_txq[0];
2449 		int i;
2450 
2451 		/* walk through the TCs and see if it falls into any of them */
2452 		for (i = 0; i < TC_MAX_QUEUE; i++, tc++) {
2453 			if ((txq - tc->offset) < tc->count)
2454 				return i;
2455 		}
2456 
2457 		/* didn't find it, just return -1 to indicate no match */
2458 		return -1;
2459 	}
2460 
2461 	return 0;
2462 }
2463 EXPORT_SYMBOL(netdev_txq_to_tc);
2464 
2465 #ifdef CONFIG_XPS
2466 struct static_key xps_needed __read_mostly;
2467 EXPORT_SYMBOL(xps_needed);
2468 struct static_key xps_rxqs_needed __read_mostly;
2469 EXPORT_SYMBOL(xps_rxqs_needed);
2470 static DEFINE_MUTEX(xps_map_mutex);
2471 #define xmap_dereference(P)		\
2472 	rcu_dereference_protected((P), lockdep_is_held(&xps_map_mutex))
2473 
2474 static bool remove_xps_queue(struct xps_dev_maps *dev_maps,
2475 			     int tci, u16 index)
2476 {
2477 	struct xps_map *map = NULL;
2478 	int pos;
2479 
2480 	if (dev_maps)
2481 		map = xmap_dereference(dev_maps->attr_map[tci]);
2482 	if (!map)
2483 		return false;
2484 
2485 	for (pos = map->len; pos--;) {
2486 		if (map->queues[pos] != index)
2487 			continue;
2488 
2489 		if (map->len > 1) {
2490 			map->queues[pos] = map->queues[--map->len];
2491 			break;
2492 		}
2493 
2494 		RCU_INIT_POINTER(dev_maps->attr_map[tci], NULL);
2495 		kfree_rcu(map, rcu);
2496 		return false;
2497 	}
2498 
2499 	return true;
2500 }
2501 
2502 static bool remove_xps_queue_cpu(struct net_device *dev,
2503 				 struct xps_dev_maps *dev_maps,
2504 				 int cpu, u16 offset, u16 count)
2505 {
2506 	int num_tc = dev->num_tc ? : 1;
2507 	bool active = false;
2508 	int tci;
2509 
2510 	for (tci = cpu * num_tc; num_tc--; tci++) {
2511 		int i, j;
2512 
2513 		for (i = count, j = offset; i--; j++) {
2514 			if (!remove_xps_queue(dev_maps, tci, j))
2515 				break;
2516 		}
2517 
2518 		active |= i < 0;
2519 	}
2520 
2521 	return active;
2522 }
2523 
2524 static void reset_xps_maps(struct net_device *dev,
2525 			   struct xps_dev_maps *dev_maps,
2526 			   bool is_rxqs_map)
2527 {
2528 	if (is_rxqs_map) {
2529 		static_key_slow_dec_cpuslocked(&xps_rxqs_needed);
2530 		RCU_INIT_POINTER(dev->xps_rxqs_map, NULL);
2531 	} else {
2532 		RCU_INIT_POINTER(dev->xps_cpus_map, NULL);
2533 	}
2534 	static_key_slow_dec_cpuslocked(&xps_needed);
2535 	kfree_rcu(dev_maps, rcu);
2536 }
2537 
2538 static void clean_xps_maps(struct net_device *dev, const unsigned long *mask,
2539 			   struct xps_dev_maps *dev_maps, unsigned int nr_ids,
2540 			   u16 offset, u16 count, bool is_rxqs_map)
2541 {
2542 	bool active = false;
2543 	int i, j;
2544 
2545 	for (j = -1; j = netif_attrmask_next(j, mask, nr_ids),
2546 	     j < nr_ids;)
2547 		active |= remove_xps_queue_cpu(dev, dev_maps, j, offset,
2548 					       count);
2549 	if (!active)
2550 		reset_xps_maps(dev, dev_maps, is_rxqs_map);
2551 
2552 	if (!is_rxqs_map) {
2553 		for (i = offset + (count - 1); count--; i--) {
2554 			netdev_queue_numa_node_write(
2555 				netdev_get_tx_queue(dev, i),
2556 				NUMA_NO_NODE);
2557 		}
2558 	}
2559 }
2560 
2561 static void netif_reset_xps_queues(struct net_device *dev, u16 offset,
2562 				   u16 count)
2563 {
2564 	const unsigned long *possible_mask = NULL;
2565 	struct xps_dev_maps *dev_maps;
2566 	unsigned int nr_ids;
2567 
2568 	if (!static_key_false(&xps_needed))
2569 		return;
2570 
2571 	cpus_read_lock();
2572 	mutex_lock(&xps_map_mutex);
2573 
2574 	if (static_key_false(&xps_rxqs_needed)) {
2575 		dev_maps = xmap_dereference(dev->xps_rxqs_map);
2576 		if (dev_maps) {
2577 			nr_ids = dev->num_rx_queues;
2578 			clean_xps_maps(dev, possible_mask, dev_maps, nr_ids,
2579 				       offset, count, true);
2580 		}
2581 	}
2582 
2583 	dev_maps = xmap_dereference(dev->xps_cpus_map);
2584 	if (!dev_maps)
2585 		goto out_no_maps;
2586 
2587 	if (num_possible_cpus() > 1)
2588 		possible_mask = cpumask_bits(cpu_possible_mask);
2589 	nr_ids = nr_cpu_ids;
2590 	clean_xps_maps(dev, possible_mask, dev_maps, nr_ids, offset, count,
2591 		       false);
2592 
2593 out_no_maps:
2594 	mutex_unlock(&xps_map_mutex);
2595 	cpus_read_unlock();
2596 }
2597 
2598 static void netif_reset_xps_queues_gt(struct net_device *dev, u16 index)
2599 {
2600 	netif_reset_xps_queues(dev, index, dev->num_tx_queues - index);
2601 }
2602 
2603 static struct xps_map *expand_xps_map(struct xps_map *map, int attr_index,
2604 				      u16 index, bool is_rxqs_map)
2605 {
2606 	struct xps_map *new_map;
2607 	int alloc_len = XPS_MIN_MAP_ALLOC;
2608 	int i, pos;
2609 
2610 	for (pos = 0; map && pos < map->len; pos++) {
2611 		if (map->queues[pos] != index)
2612 			continue;
2613 		return map;
2614 	}
2615 
2616 	/* Need to add tx-queue to this CPU's/rx-queue's existing map */
2617 	if (map) {
2618 		if (pos < map->alloc_len)
2619 			return map;
2620 
2621 		alloc_len = map->alloc_len * 2;
2622 	}
2623 
2624 	/* Need to allocate new map to store tx-queue on this CPU's/rx-queue's
2625 	 *  map
2626 	 */
2627 	if (is_rxqs_map)
2628 		new_map = kzalloc(XPS_MAP_SIZE(alloc_len), GFP_KERNEL);
2629 	else
2630 		new_map = kzalloc_node(XPS_MAP_SIZE(alloc_len), GFP_KERNEL,
2631 				       cpu_to_node(attr_index));
2632 	if (!new_map)
2633 		return NULL;
2634 
2635 	for (i = 0; i < pos; i++)
2636 		new_map->queues[i] = map->queues[i];
2637 	new_map->alloc_len = alloc_len;
2638 	new_map->len = pos;
2639 
2640 	return new_map;
2641 }
2642 
2643 /* Must be called under cpus_read_lock */
2644 int __netif_set_xps_queue(struct net_device *dev, const unsigned long *mask,
2645 			  u16 index, bool is_rxqs_map)
2646 {
2647 	const unsigned long *online_mask = NULL, *possible_mask = NULL;
2648 	struct xps_dev_maps *dev_maps, *new_dev_maps = NULL;
2649 	int i, j, tci, numa_node_id = -2;
2650 	int maps_sz, num_tc = 1, tc = 0;
2651 	struct xps_map *map, *new_map;
2652 	bool active = false;
2653 	unsigned int nr_ids;
2654 
2655 	if (dev->num_tc) {
2656 		/* Do not allow XPS on subordinate device directly */
2657 		num_tc = dev->num_tc;
2658 		if (num_tc < 0)
2659 			return -EINVAL;
2660 
2661 		/* If queue belongs to subordinate dev use its map */
2662 		dev = netdev_get_tx_queue(dev, index)->sb_dev ? : dev;
2663 
2664 		tc = netdev_txq_to_tc(dev, index);
2665 		if (tc < 0)
2666 			return -EINVAL;
2667 	}
2668 
2669 	mutex_lock(&xps_map_mutex);
2670 	if (is_rxqs_map) {
2671 		maps_sz = XPS_RXQ_DEV_MAPS_SIZE(num_tc, dev->num_rx_queues);
2672 		dev_maps = xmap_dereference(dev->xps_rxqs_map);
2673 		nr_ids = dev->num_rx_queues;
2674 	} else {
2675 		maps_sz = XPS_CPU_DEV_MAPS_SIZE(num_tc);
2676 		if (num_possible_cpus() > 1) {
2677 			online_mask = cpumask_bits(cpu_online_mask);
2678 			possible_mask = cpumask_bits(cpu_possible_mask);
2679 		}
2680 		dev_maps = xmap_dereference(dev->xps_cpus_map);
2681 		nr_ids = nr_cpu_ids;
2682 	}
2683 
2684 	if (maps_sz < L1_CACHE_BYTES)
2685 		maps_sz = L1_CACHE_BYTES;
2686 
2687 	/* allocate memory for queue storage */
2688 	for (j = -1; j = netif_attrmask_next_and(j, online_mask, mask, nr_ids),
2689 	     j < nr_ids;) {
2690 		if (!new_dev_maps)
2691 			new_dev_maps = kzalloc(maps_sz, GFP_KERNEL);
2692 		if (!new_dev_maps) {
2693 			mutex_unlock(&xps_map_mutex);
2694 			return -ENOMEM;
2695 		}
2696 
2697 		tci = j * num_tc + tc;
2698 		map = dev_maps ? xmap_dereference(dev_maps->attr_map[tci]) :
2699 				 NULL;
2700 
2701 		map = expand_xps_map(map, j, index, is_rxqs_map);
2702 		if (!map)
2703 			goto error;
2704 
2705 		RCU_INIT_POINTER(new_dev_maps->attr_map[tci], map);
2706 	}
2707 
2708 	if (!new_dev_maps)
2709 		goto out_no_new_maps;
2710 
2711 	if (!dev_maps) {
2712 		/* Increment static keys at most once per type */
2713 		static_key_slow_inc_cpuslocked(&xps_needed);
2714 		if (is_rxqs_map)
2715 			static_key_slow_inc_cpuslocked(&xps_rxqs_needed);
2716 	}
2717 
2718 	for (j = -1; j = netif_attrmask_next(j, possible_mask, nr_ids),
2719 	     j < nr_ids;) {
2720 		/* copy maps belonging to foreign traffic classes */
2721 		for (i = tc, tci = j * num_tc; dev_maps && i--; tci++) {
2722 			/* fill in the new device map from the old device map */
2723 			map = xmap_dereference(dev_maps->attr_map[tci]);
2724 			RCU_INIT_POINTER(new_dev_maps->attr_map[tci], map);
2725 		}
2726 
2727 		/* We need to explicitly update tci as prevous loop
2728 		 * could break out early if dev_maps is NULL.
2729 		 */
2730 		tci = j * num_tc + tc;
2731 
2732 		if (netif_attr_test_mask(j, mask, nr_ids) &&
2733 		    netif_attr_test_online(j, online_mask, nr_ids)) {
2734 			/* add tx-queue to CPU/rx-queue maps */
2735 			int pos = 0;
2736 
2737 			map = xmap_dereference(new_dev_maps->attr_map[tci]);
2738 			while ((pos < map->len) && (map->queues[pos] != index))
2739 				pos++;
2740 
2741 			if (pos == map->len)
2742 				map->queues[map->len++] = index;
2743 #ifdef CONFIG_NUMA
2744 			if (!is_rxqs_map) {
2745 				if (numa_node_id == -2)
2746 					numa_node_id = cpu_to_node(j);
2747 				else if (numa_node_id != cpu_to_node(j))
2748 					numa_node_id = -1;
2749 			}
2750 #endif
2751 		} else if (dev_maps) {
2752 			/* fill in the new device map from the old device map */
2753 			map = xmap_dereference(dev_maps->attr_map[tci]);
2754 			RCU_INIT_POINTER(new_dev_maps->attr_map[tci], map);
2755 		}
2756 
2757 		/* copy maps belonging to foreign traffic classes */
2758 		for (i = num_tc - tc, tci++; dev_maps && --i; tci++) {
2759 			/* fill in the new device map from the old device map */
2760 			map = xmap_dereference(dev_maps->attr_map[tci]);
2761 			RCU_INIT_POINTER(new_dev_maps->attr_map[tci], map);
2762 		}
2763 	}
2764 
2765 	if (is_rxqs_map)
2766 		rcu_assign_pointer(dev->xps_rxqs_map, new_dev_maps);
2767 	else
2768 		rcu_assign_pointer(dev->xps_cpus_map, new_dev_maps);
2769 
2770 	/* Cleanup old maps */
2771 	if (!dev_maps)
2772 		goto out_no_old_maps;
2773 
2774 	for (j = -1; j = netif_attrmask_next(j, possible_mask, nr_ids),
2775 	     j < nr_ids;) {
2776 		for (i = num_tc, tci = j * num_tc; i--; tci++) {
2777 			new_map = xmap_dereference(new_dev_maps->attr_map[tci]);
2778 			map = xmap_dereference(dev_maps->attr_map[tci]);
2779 			if (map && map != new_map)
2780 				kfree_rcu(map, rcu);
2781 		}
2782 	}
2783 
2784 	kfree_rcu(dev_maps, rcu);
2785 
2786 out_no_old_maps:
2787 	dev_maps = new_dev_maps;
2788 	active = true;
2789 
2790 out_no_new_maps:
2791 	if (!is_rxqs_map) {
2792 		/* update Tx queue numa node */
2793 		netdev_queue_numa_node_write(netdev_get_tx_queue(dev, index),
2794 					     (numa_node_id >= 0) ?
2795 					     numa_node_id : NUMA_NO_NODE);
2796 	}
2797 
2798 	if (!dev_maps)
2799 		goto out_no_maps;
2800 
2801 	/* removes tx-queue from unused CPUs/rx-queues */
2802 	for (j = -1; j = netif_attrmask_next(j, possible_mask, nr_ids),
2803 	     j < nr_ids;) {
2804 		for (i = tc, tci = j * num_tc; i--; tci++)
2805 			active |= remove_xps_queue(dev_maps, tci, index);
2806 		if (!netif_attr_test_mask(j, mask, nr_ids) ||
2807 		    !netif_attr_test_online(j, online_mask, nr_ids))
2808 			active |= remove_xps_queue(dev_maps, tci, index);
2809 		for (i = num_tc - tc, tci++; --i; tci++)
2810 			active |= remove_xps_queue(dev_maps, tci, index);
2811 	}
2812 
2813 	/* free map if not active */
2814 	if (!active)
2815 		reset_xps_maps(dev, dev_maps, is_rxqs_map);
2816 
2817 out_no_maps:
2818 	mutex_unlock(&xps_map_mutex);
2819 
2820 	return 0;
2821 error:
2822 	/* remove any maps that we added */
2823 	for (j = -1; j = netif_attrmask_next(j, possible_mask, nr_ids),
2824 	     j < nr_ids;) {
2825 		for (i = num_tc, tci = j * num_tc; i--; tci++) {
2826 			new_map = xmap_dereference(new_dev_maps->attr_map[tci]);
2827 			map = dev_maps ?
2828 			      xmap_dereference(dev_maps->attr_map[tci]) :
2829 			      NULL;
2830 			if (new_map && new_map != map)
2831 				kfree(new_map);
2832 		}
2833 	}
2834 
2835 	mutex_unlock(&xps_map_mutex);
2836 
2837 	kfree(new_dev_maps);
2838 	return -ENOMEM;
2839 }
2840 EXPORT_SYMBOL_GPL(__netif_set_xps_queue);
2841 
2842 int netif_set_xps_queue(struct net_device *dev, const struct cpumask *mask,
2843 			u16 index)
2844 {
2845 	int ret;
2846 
2847 	cpus_read_lock();
2848 	ret =  __netif_set_xps_queue(dev, cpumask_bits(mask), index, false);
2849 	cpus_read_unlock();
2850 
2851 	return ret;
2852 }
2853 EXPORT_SYMBOL(netif_set_xps_queue);
2854 
2855 #endif
2856 static void netdev_unbind_all_sb_channels(struct net_device *dev)
2857 {
2858 	struct netdev_queue *txq = &dev->_tx[dev->num_tx_queues];
2859 
2860 	/* Unbind any subordinate channels */
2861 	while (txq-- != &dev->_tx[0]) {
2862 		if (txq->sb_dev)
2863 			netdev_unbind_sb_channel(dev, txq->sb_dev);
2864 	}
2865 }
2866 
2867 void netdev_reset_tc(struct net_device *dev)
2868 {
2869 #ifdef CONFIG_XPS
2870 	netif_reset_xps_queues_gt(dev, 0);
2871 #endif
2872 	netdev_unbind_all_sb_channels(dev);
2873 
2874 	/* Reset TC configuration of device */
2875 	dev->num_tc = 0;
2876 	memset(dev->tc_to_txq, 0, sizeof(dev->tc_to_txq));
2877 	memset(dev->prio_tc_map, 0, sizeof(dev->prio_tc_map));
2878 }
2879 EXPORT_SYMBOL(netdev_reset_tc);
2880 
2881 int netdev_set_tc_queue(struct net_device *dev, u8 tc, u16 count, u16 offset)
2882 {
2883 	if (tc >= dev->num_tc)
2884 		return -EINVAL;
2885 
2886 #ifdef CONFIG_XPS
2887 	netif_reset_xps_queues(dev, offset, count);
2888 #endif
2889 	dev->tc_to_txq[tc].count = count;
2890 	dev->tc_to_txq[tc].offset = offset;
2891 	return 0;
2892 }
2893 EXPORT_SYMBOL(netdev_set_tc_queue);
2894 
2895 int netdev_set_num_tc(struct net_device *dev, u8 num_tc)
2896 {
2897 	if (num_tc > TC_MAX_QUEUE)
2898 		return -EINVAL;
2899 
2900 #ifdef CONFIG_XPS
2901 	netif_reset_xps_queues_gt(dev, 0);
2902 #endif
2903 	netdev_unbind_all_sb_channels(dev);
2904 
2905 	dev->num_tc = num_tc;
2906 	return 0;
2907 }
2908 EXPORT_SYMBOL(netdev_set_num_tc);
2909 
2910 void netdev_unbind_sb_channel(struct net_device *dev,
2911 			      struct net_device *sb_dev)
2912 {
2913 	struct netdev_queue *txq = &dev->_tx[dev->num_tx_queues];
2914 
2915 #ifdef CONFIG_XPS
2916 	netif_reset_xps_queues_gt(sb_dev, 0);
2917 #endif
2918 	memset(sb_dev->tc_to_txq, 0, sizeof(sb_dev->tc_to_txq));
2919 	memset(sb_dev->prio_tc_map, 0, sizeof(sb_dev->prio_tc_map));
2920 
2921 	while (txq-- != &dev->_tx[0]) {
2922 		if (txq->sb_dev == sb_dev)
2923 			txq->sb_dev = NULL;
2924 	}
2925 }
2926 EXPORT_SYMBOL(netdev_unbind_sb_channel);
2927 
2928 int netdev_bind_sb_channel_queue(struct net_device *dev,
2929 				 struct net_device *sb_dev,
2930 				 u8 tc, u16 count, u16 offset)
2931 {
2932 	/* Make certain the sb_dev and dev are already configured */
2933 	if (sb_dev->num_tc >= 0 || tc >= dev->num_tc)
2934 		return -EINVAL;
2935 
2936 	/* We cannot hand out queues we don't have */
2937 	if ((offset + count) > dev->real_num_tx_queues)
2938 		return -EINVAL;
2939 
2940 	/* Record the mapping */
2941 	sb_dev->tc_to_txq[tc].count = count;
2942 	sb_dev->tc_to_txq[tc].offset = offset;
2943 
2944 	/* Provide a way for Tx queue to find the tc_to_txq map or
2945 	 * XPS map for itself.
2946 	 */
2947 	while (count--)
2948 		netdev_get_tx_queue(dev, count + offset)->sb_dev = sb_dev;
2949 
2950 	return 0;
2951 }
2952 EXPORT_SYMBOL(netdev_bind_sb_channel_queue);
2953 
2954 int netdev_set_sb_channel(struct net_device *dev, u16 channel)
2955 {
2956 	/* Do not use a multiqueue device to represent a subordinate channel */
2957 	if (netif_is_multiqueue(dev))
2958 		return -ENODEV;
2959 
2960 	/* We allow channels 1 - 32767 to be used for subordinate channels.
2961 	 * Channel 0 is meant to be "native" mode and used only to represent
2962 	 * the main root device. We allow writing 0 to reset the device back
2963 	 * to normal mode after being used as a subordinate channel.
2964 	 */
2965 	if (channel > S16_MAX)
2966 		return -EINVAL;
2967 
2968 	dev->num_tc = -channel;
2969 
2970 	return 0;
2971 }
2972 EXPORT_SYMBOL(netdev_set_sb_channel);
2973 
2974 /*
2975  * Routine to help set real_num_tx_queues. To avoid skbs mapped to queues
2976  * greater than real_num_tx_queues stale skbs on the qdisc must be flushed.
2977  */
2978 int netif_set_real_num_tx_queues(struct net_device *dev, unsigned int txq)
2979 {
2980 	bool disabling;
2981 	int rc;
2982 
2983 	disabling = txq < dev->real_num_tx_queues;
2984 
2985 	if (txq < 1 || txq > dev->num_tx_queues)
2986 		return -EINVAL;
2987 
2988 	if (dev->reg_state == NETREG_REGISTERED ||
2989 	    dev->reg_state == NETREG_UNREGISTERING) {
2990 		ASSERT_RTNL();
2991 
2992 		rc = netdev_queue_update_kobjects(dev, dev->real_num_tx_queues,
2993 						  txq);
2994 		if (rc)
2995 			return rc;
2996 
2997 		if (dev->num_tc)
2998 			netif_setup_tc(dev, txq);
2999 
3000 		dev->real_num_tx_queues = txq;
3001 
3002 		if (disabling) {
3003 			synchronize_net();
3004 			qdisc_reset_all_tx_gt(dev, txq);
3005 #ifdef CONFIG_XPS
3006 			netif_reset_xps_queues_gt(dev, txq);
3007 #endif
3008 		}
3009 	} else {
3010 		dev->real_num_tx_queues = txq;
3011 	}
3012 
3013 	return 0;
3014 }
3015 EXPORT_SYMBOL(netif_set_real_num_tx_queues);
3016 
3017 #ifdef CONFIG_SYSFS
3018 /**
3019  *	netif_set_real_num_rx_queues - set actual number of RX queues used
3020  *	@dev: Network device
3021  *	@rxq: Actual number of RX queues
3022  *
3023  *	This must be called either with the rtnl_lock held or before
3024  *	registration of the net device.  Returns 0 on success, or a
3025  *	negative error code.  If called before registration, it always
3026  *	succeeds.
3027  */
3028 int netif_set_real_num_rx_queues(struct net_device *dev, unsigned int rxq)
3029 {
3030 	int rc;
3031 
3032 	if (rxq < 1 || rxq > dev->num_rx_queues)
3033 		return -EINVAL;
3034 
3035 	if (dev->reg_state == NETREG_REGISTERED) {
3036 		ASSERT_RTNL();
3037 
3038 		rc = net_rx_queue_update_kobjects(dev, dev->real_num_rx_queues,
3039 						  rxq);
3040 		if (rc)
3041 			return rc;
3042 	}
3043 
3044 	dev->real_num_rx_queues = rxq;
3045 	return 0;
3046 }
3047 EXPORT_SYMBOL(netif_set_real_num_rx_queues);
3048 #endif
3049 
3050 /**
3051  * netif_get_num_default_rss_queues - default number of RSS queues
3052  *
3053  * This routine should set an upper limit on the number of RSS queues
3054  * used by default by multiqueue devices.
3055  */
3056 int netif_get_num_default_rss_queues(void)
3057 {
3058 	return is_kdump_kernel() ?
3059 		1 : min_t(int, DEFAULT_MAX_NUM_RSS_QUEUES, num_online_cpus());
3060 }
3061 EXPORT_SYMBOL(netif_get_num_default_rss_queues);
3062 
3063 static void __netif_reschedule(struct Qdisc *q)
3064 {
3065 	struct softnet_data *sd;
3066 	unsigned long flags;
3067 
3068 	local_irq_save(flags);
3069 	sd = this_cpu_ptr(&softnet_data);
3070 	q->next_sched = NULL;
3071 	*sd->output_queue_tailp = q;
3072 	sd->output_queue_tailp = &q->next_sched;
3073 	raise_softirq_irqoff(NET_TX_SOFTIRQ);
3074 	local_irq_restore(flags);
3075 }
3076 
3077 void __netif_schedule(struct Qdisc *q)
3078 {
3079 	if (!test_and_set_bit(__QDISC_STATE_SCHED, &q->state))
3080 		__netif_reschedule(q);
3081 }
3082 EXPORT_SYMBOL(__netif_schedule);
3083 
3084 struct dev_kfree_skb_cb {
3085 	enum skb_free_reason reason;
3086 };
3087 
3088 static struct dev_kfree_skb_cb *get_kfree_skb_cb(const struct sk_buff *skb)
3089 {
3090 	return (struct dev_kfree_skb_cb *)skb->cb;
3091 }
3092 
3093 void netif_schedule_queue(struct netdev_queue *txq)
3094 {
3095 	rcu_read_lock();
3096 	if (!netif_xmit_stopped(txq)) {
3097 		struct Qdisc *q = rcu_dereference(txq->qdisc);
3098 
3099 		__netif_schedule(q);
3100 	}
3101 	rcu_read_unlock();
3102 }
3103 EXPORT_SYMBOL(netif_schedule_queue);
3104 
3105 void netif_tx_wake_queue(struct netdev_queue *dev_queue)
3106 {
3107 	if (test_and_clear_bit(__QUEUE_STATE_DRV_XOFF, &dev_queue->state)) {
3108 		struct Qdisc *q;
3109 
3110 		rcu_read_lock();
3111 		q = rcu_dereference(dev_queue->qdisc);
3112 		__netif_schedule(q);
3113 		rcu_read_unlock();
3114 	}
3115 }
3116 EXPORT_SYMBOL(netif_tx_wake_queue);
3117 
3118 void __dev_kfree_skb_irq(struct sk_buff *skb, enum skb_free_reason reason)
3119 {
3120 	unsigned long flags;
3121 
3122 	if (unlikely(!skb))
3123 		return;
3124 
3125 	if (likely(refcount_read(&skb->users) == 1)) {
3126 		smp_rmb();
3127 		refcount_set(&skb->users, 0);
3128 	} else if (likely(!refcount_dec_and_test(&skb->users))) {
3129 		return;
3130 	}
3131 	get_kfree_skb_cb(skb)->reason = reason;
3132 	local_irq_save(flags);
3133 	skb->next = __this_cpu_read(softnet_data.completion_queue);
3134 	__this_cpu_write(softnet_data.completion_queue, skb);
3135 	raise_softirq_irqoff(NET_TX_SOFTIRQ);
3136 	local_irq_restore(flags);
3137 }
3138 EXPORT_SYMBOL(__dev_kfree_skb_irq);
3139 
3140 void __dev_kfree_skb_any(struct sk_buff *skb, enum skb_free_reason reason)
3141 {
3142 	if (in_irq() || irqs_disabled())
3143 		__dev_kfree_skb_irq(skb, reason);
3144 	else
3145 		dev_kfree_skb(skb);
3146 }
3147 EXPORT_SYMBOL(__dev_kfree_skb_any);
3148 
3149 
3150 /**
3151  * netif_device_detach - mark device as removed
3152  * @dev: network device
3153  *
3154  * Mark device as removed from system and therefore no longer available.
3155  */
3156 void netif_device_detach(struct net_device *dev)
3157 {
3158 	if (test_and_clear_bit(__LINK_STATE_PRESENT, &dev->state) &&
3159 	    netif_running(dev)) {
3160 		netif_tx_stop_all_queues(dev);
3161 	}
3162 }
3163 EXPORT_SYMBOL(netif_device_detach);
3164 
3165 /**
3166  * netif_device_attach - mark device as attached
3167  * @dev: network device
3168  *
3169  * Mark device as attached from system and restart if needed.
3170  */
3171 void netif_device_attach(struct net_device *dev)
3172 {
3173 	if (!test_and_set_bit(__LINK_STATE_PRESENT, &dev->state) &&
3174 	    netif_running(dev)) {
3175 		netif_tx_wake_all_queues(dev);
3176 		__netdev_watchdog_up(dev);
3177 	}
3178 }
3179 EXPORT_SYMBOL(netif_device_attach);
3180 
3181 /*
3182  * Returns a Tx hash based on the given packet descriptor a Tx queues' number
3183  * to be used as a distribution range.
3184  */
3185 static u16 skb_tx_hash(const struct net_device *dev,
3186 		       const struct net_device *sb_dev,
3187 		       struct sk_buff *skb)
3188 {
3189 	u32 hash;
3190 	u16 qoffset = 0;
3191 	u16 qcount = dev->real_num_tx_queues;
3192 
3193 	if (dev->num_tc) {
3194 		u8 tc = netdev_get_prio_tc_map(dev, skb->priority);
3195 
3196 		qoffset = sb_dev->tc_to_txq[tc].offset;
3197 		qcount = sb_dev->tc_to_txq[tc].count;
3198 	}
3199 
3200 	if (skb_rx_queue_recorded(skb)) {
3201 		hash = skb_get_rx_queue(skb);
3202 		if (hash >= qoffset)
3203 			hash -= qoffset;
3204 		while (unlikely(hash >= qcount))
3205 			hash -= qcount;
3206 		return hash + qoffset;
3207 	}
3208 
3209 	return (u16) reciprocal_scale(skb_get_hash(skb), qcount) + qoffset;
3210 }
3211 
3212 static void skb_warn_bad_offload(const struct sk_buff *skb)
3213 {
3214 	static const netdev_features_t null_features;
3215 	struct net_device *dev = skb->dev;
3216 	const char *name = "";
3217 
3218 	if (!net_ratelimit())
3219 		return;
3220 
3221 	if (dev) {
3222 		if (dev->dev.parent)
3223 			name = dev_driver_string(dev->dev.parent);
3224 		else
3225 			name = netdev_name(dev);
3226 	}
3227 	skb_dump(KERN_WARNING, skb, false);
3228 	WARN(1, "%s: caps=(%pNF, %pNF)\n",
3229 	     name, dev ? &dev->features : &null_features,
3230 	     skb->sk ? &skb->sk->sk_route_caps : &null_features);
3231 }
3232 
3233 /*
3234  * Invalidate hardware checksum when packet is to be mangled, and
3235  * complete checksum manually on outgoing path.
3236  */
3237 int skb_checksum_help(struct sk_buff *skb)
3238 {
3239 	__wsum csum;
3240 	int ret = 0, offset;
3241 
3242 	if (skb->ip_summed == CHECKSUM_COMPLETE)
3243 		goto out_set_summed;
3244 
3245 	if (unlikely(skb_is_gso(skb))) {
3246 		skb_warn_bad_offload(skb);
3247 		return -EINVAL;
3248 	}
3249 
3250 	/* Before computing a checksum, we should make sure no frag could
3251 	 * be modified by an external entity : checksum could be wrong.
3252 	 */
3253 	if (skb_has_shared_frag(skb)) {
3254 		ret = __skb_linearize(skb);
3255 		if (ret)
3256 			goto out;
3257 	}
3258 
3259 	offset = skb_checksum_start_offset(skb);
3260 	BUG_ON(offset >= skb_headlen(skb));
3261 	csum = skb_checksum(skb, offset, skb->len - offset, 0);
3262 
3263 	offset += skb->csum_offset;
3264 	BUG_ON(offset + sizeof(__sum16) > skb_headlen(skb));
3265 
3266 	ret = skb_ensure_writable(skb, offset + sizeof(__sum16));
3267 	if (ret)
3268 		goto out;
3269 
3270 	*(__sum16 *)(skb->data + offset) = csum_fold(csum) ?: CSUM_MANGLED_0;
3271 out_set_summed:
3272 	skb->ip_summed = CHECKSUM_NONE;
3273 out:
3274 	return ret;
3275 }
3276 EXPORT_SYMBOL(skb_checksum_help);
3277 
3278 int skb_crc32c_csum_help(struct sk_buff *skb)
3279 {
3280 	__le32 crc32c_csum;
3281 	int ret = 0, offset, start;
3282 
3283 	if (skb->ip_summed != CHECKSUM_PARTIAL)
3284 		goto out;
3285 
3286 	if (unlikely(skb_is_gso(skb)))
3287 		goto out;
3288 
3289 	/* Before computing a checksum, we should make sure no frag could
3290 	 * be modified by an external entity : checksum could be wrong.
3291 	 */
3292 	if (unlikely(skb_has_shared_frag(skb))) {
3293 		ret = __skb_linearize(skb);
3294 		if (ret)
3295 			goto out;
3296 	}
3297 	start = skb_checksum_start_offset(skb);
3298 	offset = start + offsetof(struct sctphdr, checksum);
3299 	if (WARN_ON_ONCE(offset >= skb_headlen(skb))) {
3300 		ret = -EINVAL;
3301 		goto out;
3302 	}
3303 
3304 	ret = skb_ensure_writable(skb, offset + sizeof(__le32));
3305 	if (ret)
3306 		goto out;
3307 
3308 	crc32c_csum = cpu_to_le32(~__skb_checksum(skb, start,
3309 						  skb->len - start, ~(__u32)0,
3310 						  crc32c_csum_stub));
3311 	*(__le32 *)(skb->data + offset) = crc32c_csum;
3312 	skb->ip_summed = CHECKSUM_NONE;
3313 	skb->csum_not_inet = 0;
3314 out:
3315 	return ret;
3316 }
3317 
3318 __be16 skb_network_protocol(struct sk_buff *skb, int *depth)
3319 {
3320 	__be16 type = skb->protocol;
3321 
3322 	/* Tunnel gso handlers can set protocol to ethernet. */
3323 	if (type == htons(ETH_P_TEB)) {
3324 		struct ethhdr *eth;
3325 
3326 		if (unlikely(!pskb_may_pull(skb, sizeof(struct ethhdr))))
3327 			return 0;
3328 
3329 		eth = (struct ethhdr *)skb->data;
3330 		type = eth->h_proto;
3331 	}
3332 
3333 	return __vlan_get_protocol(skb, type, depth);
3334 }
3335 
3336 /**
3337  *	skb_mac_gso_segment - mac layer segmentation handler.
3338  *	@skb: buffer to segment
3339  *	@features: features for the output path (see dev->features)
3340  */
3341 struct sk_buff *skb_mac_gso_segment(struct sk_buff *skb,
3342 				    netdev_features_t features)
3343 {
3344 	struct sk_buff *segs = ERR_PTR(-EPROTONOSUPPORT);
3345 	struct packet_offload *ptype;
3346 	int vlan_depth = skb->mac_len;
3347 	__be16 type = skb_network_protocol(skb, &vlan_depth);
3348 
3349 	if (unlikely(!type))
3350 		return ERR_PTR(-EINVAL);
3351 
3352 	__skb_pull(skb, vlan_depth);
3353 
3354 	rcu_read_lock();
3355 	list_for_each_entry_rcu(ptype, &offload_base, list) {
3356 		if (ptype->type == type && ptype->callbacks.gso_segment) {
3357 			segs = ptype->callbacks.gso_segment(skb, features);
3358 			break;
3359 		}
3360 	}
3361 	rcu_read_unlock();
3362 
3363 	__skb_push(skb, skb->data - skb_mac_header(skb));
3364 
3365 	return segs;
3366 }
3367 EXPORT_SYMBOL(skb_mac_gso_segment);
3368 
3369 
3370 /* openvswitch calls this on rx path, so we need a different check.
3371  */
3372 static inline bool skb_needs_check(struct sk_buff *skb, bool tx_path)
3373 {
3374 	if (tx_path)
3375 		return skb->ip_summed != CHECKSUM_PARTIAL &&
3376 		       skb->ip_summed != CHECKSUM_UNNECESSARY;
3377 
3378 	return skb->ip_summed == CHECKSUM_NONE;
3379 }
3380 
3381 /**
3382  *	__skb_gso_segment - Perform segmentation on skb.
3383  *	@skb: buffer to segment
3384  *	@features: features for the output path (see dev->features)
3385  *	@tx_path: whether it is called in TX path
3386  *
3387  *	This function segments the given skb and returns a list of segments.
3388  *
3389  *	It may return NULL if the skb requires no segmentation.  This is
3390  *	only possible when GSO is used for verifying header integrity.
3391  *
3392  *	Segmentation preserves SKB_GSO_CB_OFFSET bytes of previous skb cb.
3393  */
3394 struct sk_buff *__skb_gso_segment(struct sk_buff *skb,
3395 				  netdev_features_t features, bool tx_path)
3396 {
3397 	struct sk_buff *segs;
3398 
3399 	if (unlikely(skb_needs_check(skb, tx_path))) {
3400 		int err;
3401 
3402 		/* We're going to init ->check field in TCP or UDP header */
3403 		err = skb_cow_head(skb, 0);
3404 		if (err < 0)
3405 			return ERR_PTR(err);
3406 	}
3407 
3408 	/* Only report GSO partial support if it will enable us to
3409 	 * support segmentation on this frame without needing additional
3410 	 * work.
3411 	 */
3412 	if (features & NETIF_F_GSO_PARTIAL) {
3413 		netdev_features_t partial_features = NETIF_F_GSO_ROBUST;
3414 		struct net_device *dev = skb->dev;
3415 
3416 		partial_features |= dev->features & dev->gso_partial_features;
3417 		if (!skb_gso_ok(skb, features | partial_features))
3418 			features &= ~NETIF_F_GSO_PARTIAL;
3419 	}
3420 
3421 	BUILD_BUG_ON(SKB_GSO_CB_OFFSET +
3422 		     sizeof(*SKB_GSO_CB(skb)) > sizeof(skb->cb));
3423 
3424 	SKB_GSO_CB(skb)->mac_offset = skb_headroom(skb);
3425 	SKB_GSO_CB(skb)->encap_level = 0;
3426 
3427 	skb_reset_mac_header(skb);
3428 	skb_reset_mac_len(skb);
3429 
3430 	segs = skb_mac_gso_segment(skb, features);
3431 
3432 	if (segs != skb && unlikely(skb_needs_check(skb, tx_path) && !IS_ERR(segs)))
3433 		skb_warn_bad_offload(skb);
3434 
3435 	return segs;
3436 }
3437 EXPORT_SYMBOL(__skb_gso_segment);
3438 
3439 /* Take action when hardware reception checksum errors are detected. */
3440 #ifdef CONFIG_BUG
3441 void netdev_rx_csum_fault(struct net_device *dev, struct sk_buff *skb)
3442 {
3443 	if (net_ratelimit()) {
3444 		pr_err("%s: hw csum failure\n", dev ? dev->name : "<unknown>");
3445 		skb_dump(KERN_ERR, skb, true);
3446 		dump_stack();
3447 	}
3448 }
3449 EXPORT_SYMBOL(netdev_rx_csum_fault);
3450 #endif
3451 
3452 /* XXX: check that highmem exists at all on the given machine. */
3453 static int illegal_highdma(struct net_device *dev, struct sk_buff *skb)
3454 {
3455 #ifdef CONFIG_HIGHMEM
3456 	int i;
3457 
3458 	if (!(dev->features & NETIF_F_HIGHDMA)) {
3459 		for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
3460 			skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
3461 
3462 			if (PageHighMem(skb_frag_page(frag)))
3463 				return 1;
3464 		}
3465 	}
3466 #endif
3467 	return 0;
3468 }
3469 
3470 /* If MPLS offload request, verify we are testing hardware MPLS features
3471  * instead of standard features for the netdev.
3472  */
3473 #if IS_ENABLED(CONFIG_NET_MPLS_GSO)
3474 static netdev_features_t net_mpls_features(struct sk_buff *skb,
3475 					   netdev_features_t features,
3476 					   __be16 type)
3477 {
3478 	if (eth_p_mpls(type))
3479 		features &= skb->dev->mpls_features;
3480 
3481 	return features;
3482 }
3483 #else
3484 static netdev_features_t net_mpls_features(struct sk_buff *skb,
3485 					   netdev_features_t features,
3486 					   __be16 type)
3487 {
3488 	return features;
3489 }
3490 #endif
3491 
3492 static netdev_features_t harmonize_features(struct sk_buff *skb,
3493 	netdev_features_t features)
3494 {
3495 	__be16 type;
3496 
3497 	type = skb_network_protocol(skb, NULL);
3498 	features = net_mpls_features(skb, features, type);
3499 
3500 	if (skb->ip_summed != CHECKSUM_NONE &&
3501 	    !can_checksum_protocol(features, type)) {
3502 		features &= ~(NETIF_F_CSUM_MASK | NETIF_F_GSO_MASK);
3503 	}
3504 	if (illegal_highdma(skb->dev, skb))
3505 		features &= ~NETIF_F_SG;
3506 
3507 	return features;
3508 }
3509 
3510 netdev_features_t passthru_features_check(struct sk_buff *skb,
3511 					  struct net_device *dev,
3512 					  netdev_features_t features)
3513 {
3514 	return features;
3515 }
3516 EXPORT_SYMBOL(passthru_features_check);
3517 
3518 static netdev_features_t dflt_features_check(struct sk_buff *skb,
3519 					     struct net_device *dev,
3520 					     netdev_features_t features)
3521 {
3522 	return vlan_features_check(skb, features);
3523 }
3524 
3525 static netdev_features_t gso_features_check(const struct sk_buff *skb,
3526 					    struct net_device *dev,
3527 					    netdev_features_t features)
3528 {
3529 	u16 gso_segs = skb_shinfo(skb)->gso_segs;
3530 
3531 	if (gso_segs > dev->gso_max_segs)
3532 		return features & ~NETIF_F_GSO_MASK;
3533 
3534 	if (!skb_shinfo(skb)->gso_type) {
3535 		skb_warn_bad_offload(skb);
3536 		return features & ~NETIF_F_GSO_MASK;
3537 	}
3538 
3539 	/* Support for GSO partial features requires software
3540 	 * intervention before we can actually process the packets
3541 	 * so we need to strip support for any partial features now
3542 	 * and we can pull them back in after we have partially
3543 	 * segmented the frame.
3544 	 */
3545 	if (!(skb_shinfo(skb)->gso_type & SKB_GSO_PARTIAL))
3546 		features &= ~dev->gso_partial_features;
3547 
3548 	/* Make sure to clear the IPv4 ID mangling feature if the
3549 	 * IPv4 header has the potential to be fragmented.
3550 	 */
3551 	if (skb_shinfo(skb)->gso_type & SKB_GSO_TCPV4) {
3552 		struct iphdr *iph = skb->encapsulation ?
3553 				    inner_ip_hdr(skb) : ip_hdr(skb);
3554 
3555 		if (!(iph->frag_off & htons(IP_DF)))
3556 			features &= ~NETIF_F_TSO_MANGLEID;
3557 	}
3558 
3559 	return features;
3560 }
3561 
3562 netdev_features_t netif_skb_features(struct sk_buff *skb)
3563 {
3564 	struct net_device *dev = skb->dev;
3565 	netdev_features_t features = dev->features;
3566 
3567 	if (skb_is_gso(skb))
3568 		features = gso_features_check(skb, dev, features);
3569 
3570 	/* If encapsulation offload request, verify we are testing
3571 	 * hardware encapsulation features instead of standard
3572 	 * features for the netdev
3573 	 */
3574 	if (skb->encapsulation)
3575 		features &= dev->hw_enc_features;
3576 
3577 	if (skb_vlan_tagged(skb))
3578 		features = netdev_intersect_features(features,
3579 						     dev->vlan_features |
3580 						     NETIF_F_HW_VLAN_CTAG_TX |
3581 						     NETIF_F_HW_VLAN_STAG_TX);
3582 
3583 	if (dev->netdev_ops->ndo_features_check)
3584 		features &= dev->netdev_ops->ndo_features_check(skb, dev,
3585 								features);
3586 	else
3587 		features &= dflt_features_check(skb, dev, features);
3588 
3589 	return harmonize_features(skb, features);
3590 }
3591 EXPORT_SYMBOL(netif_skb_features);
3592 
3593 static int xmit_one(struct sk_buff *skb, struct net_device *dev,
3594 		    struct netdev_queue *txq, bool more)
3595 {
3596 	unsigned int len;
3597 	int rc;
3598 
3599 	if (dev_nit_active(dev))
3600 		dev_queue_xmit_nit(skb, dev);
3601 
3602 	len = skb->len;
3603 	PRANDOM_ADD_NOISE(skb, dev, txq, len + jiffies);
3604 	trace_net_dev_start_xmit(skb, dev);
3605 	rc = netdev_start_xmit(skb, dev, txq, more);
3606 	trace_net_dev_xmit(skb, rc, dev, len);
3607 
3608 	return rc;
3609 }
3610 
3611 struct sk_buff *dev_hard_start_xmit(struct sk_buff *first, struct net_device *dev,
3612 				    struct netdev_queue *txq, int *ret)
3613 {
3614 	struct sk_buff *skb = first;
3615 	int rc = NETDEV_TX_OK;
3616 
3617 	while (skb) {
3618 		struct sk_buff *next = skb->next;
3619 
3620 		skb_mark_not_on_list(skb);
3621 		rc = xmit_one(skb, dev, txq, next != NULL);
3622 		if (unlikely(!dev_xmit_complete(rc))) {
3623 			skb->next = next;
3624 			goto out;
3625 		}
3626 
3627 		skb = next;
3628 		if (netif_tx_queue_stopped(txq) && skb) {
3629 			rc = NETDEV_TX_BUSY;
3630 			break;
3631 		}
3632 	}
3633 
3634 out:
3635 	*ret = rc;
3636 	return skb;
3637 }
3638 
3639 static struct sk_buff *validate_xmit_vlan(struct sk_buff *skb,
3640 					  netdev_features_t features)
3641 {
3642 	if (skb_vlan_tag_present(skb) &&
3643 	    !vlan_hw_offload_capable(features, skb->vlan_proto))
3644 		skb = __vlan_hwaccel_push_inside(skb);
3645 	return skb;
3646 }
3647 
3648 int skb_csum_hwoffload_help(struct sk_buff *skb,
3649 			    const netdev_features_t features)
3650 {
3651 	if (unlikely(skb_csum_is_sctp(skb)))
3652 		return !!(features & NETIF_F_SCTP_CRC) ? 0 :
3653 			skb_crc32c_csum_help(skb);
3654 
3655 	if (features & NETIF_F_HW_CSUM)
3656 		return 0;
3657 
3658 	if (features & (NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM)) {
3659 		switch (skb->csum_offset) {
3660 		case offsetof(struct tcphdr, check):
3661 		case offsetof(struct udphdr, check):
3662 			return 0;
3663 		}
3664 	}
3665 
3666 	return skb_checksum_help(skb);
3667 }
3668 EXPORT_SYMBOL(skb_csum_hwoffload_help);
3669 
3670 static struct sk_buff *validate_xmit_skb(struct sk_buff *skb, struct net_device *dev, bool *again)
3671 {
3672 	netdev_features_t features;
3673 
3674 	features = netif_skb_features(skb);
3675 	skb = validate_xmit_vlan(skb, features);
3676 	if (unlikely(!skb))
3677 		goto out_null;
3678 
3679 	skb = sk_validate_xmit_skb(skb, dev);
3680 	if (unlikely(!skb))
3681 		goto out_null;
3682 
3683 	if (netif_needs_gso(skb, features)) {
3684 		struct sk_buff *segs;
3685 
3686 		segs = skb_gso_segment(skb, features);
3687 		if (IS_ERR(segs)) {
3688 			goto out_kfree_skb;
3689 		} else if (segs) {
3690 			consume_skb(skb);
3691 			skb = segs;
3692 		}
3693 	} else {
3694 		if (skb_needs_linearize(skb, features) &&
3695 		    __skb_linearize(skb))
3696 			goto out_kfree_skb;
3697 
3698 		/* If packet is not checksummed and device does not
3699 		 * support checksumming for this protocol, complete
3700 		 * checksumming here.
3701 		 */
3702 		if (skb->ip_summed == CHECKSUM_PARTIAL) {
3703 			if (skb->encapsulation)
3704 				skb_set_inner_transport_header(skb,
3705 							       skb_checksum_start_offset(skb));
3706 			else
3707 				skb_set_transport_header(skb,
3708 							 skb_checksum_start_offset(skb));
3709 			if (skb_csum_hwoffload_help(skb, features))
3710 				goto out_kfree_skb;
3711 		}
3712 	}
3713 
3714 	skb = validate_xmit_xfrm(skb, features, again);
3715 
3716 	return skb;
3717 
3718 out_kfree_skb:
3719 	kfree_skb(skb);
3720 out_null:
3721 	atomic_long_inc(&dev->tx_dropped);
3722 	return NULL;
3723 }
3724 
3725 struct sk_buff *validate_xmit_skb_list(struct sk_buff *skb, struct net_device *dev, bool *again)
3726 {
3727 	struct sk_buff *next, *head = NULL, *tail;
3728 
3729 	for (; skb != NULL; skb = next) {
3730 		next = skb->next;
3731 		skb_mark_not_on_list(skb);
3732 
3733 		/* in case skb wont be segmented, point to itself */
3734 		skb->prev = skb;
3735 
3736 		skb = validate_xmit_skb(skb, dev, again);
3737 		if (!skb)
3738 			continue;
3739 
3740 		if (!head)
3741 			head = skb;
3742 		else
3743 			tail->next = skb;
3744 		/* If skb was segmented, skb->prev points to
3745 		 * the last segment. If not, it still contains skb.
3746 		 */
3747 		tail = skb->prev;
3748 	}
3749 	return head;
3750 }
3751 EXPORT_SYMBOL_GPL(validate_xmit_skb_list);
3752 
3753 static void qdisc_pkt_len_init(struct sk_buff *skb)
3754 {
3755 	const struct skb_shared_info *shinfo = skb_shinfo(skb);
3756 
3757 	qdisc_skb_cb(skb)->pkt_len = skb->len;
3758 
3759 	/* To get more precise estimation of bytes sent on wire,
3760 	 * we add to pkt_len the headers size of all segments
3761 	 */
3762 	if (shinfo->gso_size && skb_transport_header_was_set(skb)) {
3763 		unsigned int hdr_len;
3764 		u16 gso_segs = shinfo->gso_segs;
3765 
3766 		/* mac layer + network layer */
3767 		hdr_len = skb_transport_header(skb) - skb_mac_header(skb);
3768 
3769 		/* + transport layer */
3770 		if (likely(shinfo->gso_type & (SKB_GSO_TCPV4 | SKB_GSO_TCPV6))) {
3771 			const struct tcphdr *th;
3772 			struct tcphdr _tcphdr;
3773 
3774 			th = skb_header_pointer(skb, skb_transport_offset(skb),
3775 						sizeof(_tcphdr), &_tcphdr);
3776 			if (likely(th))
3777 				hdr_len += __tcp_hdrlen(th);
3778 		} else {
3779 			struct udphdr _udphdr;
3780 
3781 			if (skb_header_pointer(skb, skb_transport_offset(skb),
3782 					       sizeof(_udphdr), &_udphdr))
3783 				hdr_len += sizeof(struct udphdr);
3784 		}
3785 
3786 		if (shinfo->gso_type & SKB_GSO_DODGY)
3787 			gso_segs = DIV_ROUND_UP(skb->len - hdr_len,
3788 						shinfo->gso_size);
3789 
3790 		qdisc_skb_cb(skb)->pkt_len += (gso_segs - 1) * hdr_len;
3791 	}
3792 }
3793 
3794 static inline int __dev_xmit_skb(struct sk_buff *skb, struct Qdisc *q,
3795 				 struct net_device *dev,
3796 				 struct netdev_queue *txq)
3797 {
3798 	spinlock_t *root_lock = qdisc_lock(q);
3799 	struct sk_buff *to_free = NULL;
3800 	bool contended;
3801 	int rc;
3802 
3803 	qdisc_calculate_pkt_len(skb, q);
3804 
3805 	if (q->flags & TCQ_F_NOLOCK) {
3806 		rc = q->enqueue(skb, q, &to_free) & NET_XMIT_MASK;
3807 		qdisc_run(q);
3808 
3809 		if (unlikely(to_free))
3810 			kfree_skb_list(to_free);
3811 		return rc;
3812 	}
3813 
3814 	/*
3815 	 * Heuristic to force contended enqueues to serialize on a
3816 	 * separate lock before trying to get qdisc main lock.
3817 	 * This permits qdisc->running owner to get the lock more
3818 	 * often and dequeue packets faster.
3819 	 */
3820 	contended = qdisc_is_running(q);
3821 	if (unlikely(contended))
3822 		spin_lock(&q->busylock);
3823 
3824 	spin_lock(root_lock);
3825 	if (unlikely(test_bit(__QDISC_STATE_DEACTIVATED, &q->state))) {
3826 		__qdisc_drop(skb, &to_free);
3827 		rc = NET_XMIT_DROP;
3828 	} else if ((q->flags & TCQ_F_CAN_BYPASS) && !qdisc_qlen(q) &&
3829 		   qdisc_run_begin(q)) {
3830 		/*
3831 		 * This is a work-conserving queue; there are no old skbs
3832 		 * waiting to be sent out; and the qdisc is not running -
3833 		 * xmit the skb directly.
3834 		 */
3835 
3836 		qdisc_bstats_update(q, skb);
3837 
3838 		if (sch_direct_xmit(skb, q, dev, txq, root_lock, true)) {
3839 			if (unlikely(contended)) {
3840 				spin_unlock(&q->busylock);
3841 				contended = false;
3842 			}
3843 			__qdisc_run(q);
3844 		}
3845 
3846 		qdisc_run_end(q);
3847 		rc = NET_XMIT_SUCCESS;
3848 	} else {
3849 		rc = q->enqueue(skb, q, &to_free) & NET_XMIT_MASK;
3850 		if (qdisc_run_begin(q)) {
3851 			if (unlikely(contended)) {
3852 				spin_unlock(&q->busylock);
3853 				contended = false;
3854 			}
3855 			__qdisc_run(q);
3856 			qdisc_run_end(q);
3857 		}
3858 	}
3859 	spin_unlock(root_lock);
3860 	if (unlikely(to_free))
3861 		kfree_skb_list(to_free);
3862 	if (unlikely(contended))
3863 		spin_unlock(&q->busylock);
3864 	return rc;
3865 }
3866 
3867 #if IS_ENABLED(CONFIG_CGROUP_NET_PRIO)
3868 static void skb_update_prio(struct sk_buff *skb)
3869 {
3870 	const struct netprio_map *map;
3871 	const struct sock *sk;
3872 	unsigned int prioidx;
3873 
3874 	if (skb->priority)
3875 		return;
3876 	map = rcu_dereference_bh(skb->dev->priomap);
3877 	if (!map)
3878 		return;
3879 	sk = skb_to_full_sk(skb);
3880 	if (!sk)
3881 		return;
3882 
3883 	prioidx = sock_cgroup_prioidx(&sk->sk_cgrp_data);
3884 
3885 	if (prioidx < map->priomap_len)
3886 		skb->priority = map->priomap[prioidx];
3887 }
3888 #else
3889 #define skb_update_prio(skb)
3890 #endif
3891 
3892 /**
3893  *	dev_loopback_xmit - loop back @skb
3894  *	@net: network namespace this loopback is happening in
3895  *	@sk:  sk needed to be a netfilter okfn
3896  *	@skb: buffer to transmit
3897  */
3898 int dev_loopback_xmit(struct net *net, struct sock *sk, struct sk_buff *skb)
3899 {
3900 	skb_reset_mac_header(skb);
3901 	__skb_pull(skb, skb_network_offset(skb));
3902 	skb->pkt_type = PACKET_LOOPBACK;
3903 	skb->ip_summed = CHECKSUM_UNNECESSARY;
3904 	WARN_ON(!skb_dst(skb));
3905 	skb_dst_force(skb);
3906 	netif_rx_ni(skb);
3907 	return 0;
3908 }
3909 EXPORT_SYMBOL(dev_loopback_xmit);
3910 
3911 #ifdef CONFIG_NET_EGRESS
3912 static struct sk_buff *
3913 sch_handle_egress(struct sk_buff *skb, int *ret, struct net_device *dev)
3914 {
3915 	struct mini_Qdisc *miniq = rcu_dereference_bh(dev->miniq_egress);
3916 	struct tcf_result cl_res;
3917 
3918 	if (!miniq)
3919 		return skb;
3920 
3921 	/* qdisc_skb_cb(skb)->pkt_len was already set by the caller. */
3922 	qdisc_skb_cb(skb)->mru = 0;
3923 	qdisc_skb_cb(skb)->post_ct = false;
3924 	mini_qdisc_bstats_cpu_update(miniq, skb);
3925 
3926 	switch (tcf_classify(skb, miniq->filter_list, &cl_res, false)) {
3927 	case TC_ACT_OK:
3928 	case TC_ACT_RECLASSIFY:
3929 		skb->tc_index = TC_H_MIN(cl_res.classid);
3930 		break;
3931 	case TC_ACT_SHOT:
3932 		mini_qdisc_qstats_cpu_drop(miniq);
3933 		*ret = NET_XMIT_DROP;
3934 		kfree_skb(skb);
3935 		return NULL;
3936 	case TC_ACT_STOLEN:
3937 	case TC_ACT_QUEUED:
3938 	case TC_ACT_TRAP:
3939 		*ret = NET_XMIT_SUCCESS;
3940 		consume_skb(skb);
3941 		return NULL;
3942 	case TC_ACT_REDIRECT:
3943 		/* No need to push/pop skb's mac_header here on egress! */
3944 		skb_do_redirect(skb);
3945 		*ret = NET_XMIT_SUCCESS;
3946 		return NULL;
3947 	default:
3948 		break;
3949 	}
3950 
3951 	return skb;
3952 }
3953 #endif /* CONFIG_NET_EGRESS */
3954 
3955 #ifdef CONFIG_XPS
3956 static int __get_xps_queue_idx(struct net_device *dev, struct sk_buff *skb,
3957 			       struct xps_dev_maps *dev_maps, unsigned int tci)
3958 {
3959 	struct xps_map *map;
3960 	int queue_index = -1;
3961 
3962 	if (dev->num_tc) {
3963 		tci *= dev->num_tc;
3964 		tci += netdev_get_prio_tc_map(dev, skb->priority);
3965 	}
3966 
3967 	map = rcu_dereference(dev_maps->attr_map[tci]);
3968 	if (map) {
3969 		if (map->len == 1)
3970 			queue_index = map->queues[0];
3971 		else
3972 			queue_index = map->queues[reciprocal_scale(
3973 						skb_get_hash(skb), map->len)];
3974 		if (unlikely(queue_index >= dev->real_num_tx_queues))
3975 			queue_index = -1;
3976 	}
3977 	return queue_index;
3978 }
3979 #endif
3980 
3981 static int get_xps_queue(struct net_device *dev, struct net_device *sb_dev,
3982 			 struct sk_buff *skb)
3983 {
3984 #ifdef CONFIG_XPS
3985 	struct xps_dev_maps *dev_maps;
3986 	struct sock *sk = skb->sk;
3987 	int queue_index = -1;
3988 
3989 	if (!static_key_false(&xps_needed))
3990 		return -1;
3991 
3992 	rcu_read_lock();
3993 	if (!static_key_false(&xps_rxqs_needed))
3994 		goto get_cpus_map;
3995 
3996 	dev_maps = rcu_dereference(sb_dev->xps_rxqs_map);
3997 	if (dev_maps) {
3998 		int tci = sk_rx_queue_get(sk);
3999 
4000 		if (tci >= 0 && tci < dev->num_rx_queues)
4001 			queue_index = __get_xps_queue_idx(dev, skb, dev_maps,
4002 							  tci);
4003 	}
4004 
4005 get_cpus_map:
4006 	if (queue_index < 0) {
4007 		dev_maps = rcu_dereference(sb_dev->xps_cpus_map);
4008 		if (dev_maps) {
4009 			unsigned int tci = skb->sender_cpu - 1;
4010 
4011 			queue_index = __get_xps_queue_idx(dev, skb, dev_maps,
4012 							  tci);
4013 		}
4014 	}
4015 	rcu_read_unlock();
4016 
4017 	return queue_index;
4018 #else
4019 	return -1;
4020 #endif
4021 }
4022 
4023 u16 dev_pick_tx_zero(struct net_device *dev, struct sk_buff *skb,
4024 		     struct net_device *sb_dev)
4025 {
4026 	return 0;
4027 }
4028 EXPORT_SYMBOL(dev_pick_tx_zero);
4029 
4030 u16 dev_pick_tx_cpu_id(struct net_device *dev, struct sk_buff *skb,
4031 		       struct net_device *sb_dev)
4032 {
4033 	return (u16)raw_smp_processor_id() % dev->real_num_tx_queues;
4034 }
4035 EXPORT_SYMBOL(dev_pick_tx_cpu_id);
4036 
4037 u16 netdev_pick_tx(struct net_device *dev, struct sk_buff *skb,
4038 		     struct net_device *sb_dev)
4039 {
4040 	struct sock *sk = skb->sk;
4041 	int queue_index = sk_tx_queue_get(sk);
4042 
4043 	sb_dev = sb_dev ? : dev;
4044 
4045 	if (queue_index < 0 || skb->ooo_okay ||
4046 	    queue_index >= dev->real_num_tx_queues) {
4047 		int new_index = get_xps_queue(dev, sb_dev, skb);
4048 
4049 		if (new_index < 0)
4050 			new_index = skb_tx_hash(dev, sb_dev, skb);
4051 
4052 		if (queue_index != new_index && sk &&
4053 		    sk_fullsock(sk) &&
4054 		    rcu_access_pointer(sk->sk_dst_cache))
4055 			sk_tx_queue_set(sk, new_index);
4056 
4057 		queue_index = new_index;
4058 	}
4059 
4060 	return queue_index;
4061 }
4062 EXPORT_SYMBOL(netdev_pick_tx);
4063 
4064 struct netdev_queue *netdev_core_pick_tx(struct net_device *dev,
4065 					 struct sk_buff *skb,
4066 					 struct net_device *sb_dev)
4067 {
4068 	int queue_index = 0;
4069 
4070 #ifdef CONFIG_XPS
4071 	u32 sender_cpu = skb->sender_cpu - 1;
4072 
4073 	if (sender_cpu >= (u32)NR_CPUS)
4074 		skb->sender_cpu = raw_smp_processor_id() + 1;
4075 #endif
4076 
4077 	if (dev->real_num_tx_queues != 1) {
4078 		const struct net_device_ops *ops = dev->netdev_ops;
4079 
4080 		if (ops->ndo_select_queue)
4081 			queue_index = ops->ndo_select_queue(dev, skb, sb_dev);
4082 		else
4083 			queue_index = netdev_pick_tx(dev, skb, sb_dev);
4084 
4085 		queue_index = netdev_cap_txqueue(dev, queue_index);
4086 	}
4087 
4088 	skb_set_queue_mapping(skb, queue_index);
4089 	return netdev_get_tx_queue(dev, queue_index);
4090 }
4091 
4092 /**
4093  *	__dev_queue_xmit - transmit a buffer
4094  *	@skb: buffer to transmit
4095  *	@sb_dev: suboordinate device used for L2 forwarding offload
4096  *
4097  *	Queue a buffer for transmission to a network device. The caller must
4098  *	have set the device and priority and built the buffer before calling
4099  *	this function. The function can be called from an interrupt.
4100  *
4101  *	A negative errno code is returned on a failure. A success does not
4102  *	guarantee the frame will be transmitted as it may be dropped due
4103  *	to congestion or traffic shaping.
4104  *
4105  * -----------------------------------------------------------------------------------
4106  *      I notice this method can also return errors from the queue disciplines,
4107  *      including NET_XMIT_DROP, which is a positive value.  So, errors can also
4108  *      be positive.
4109  *
4110  *      Regardless of the return value, the skb is consumed, so it is currently
4111  *      difficult to retry a send to this method.  (You can bump the ref count
4112  *      before sending to hold a reference for retry if you are careful.)
4113  *
4114  *      When calling this method, interrupts MUST be enabled.  This is because
4115  *      the BH enable code must have IRQs enabled so that it will not deadlock.
4116  *          --BLG
4117  */
4118 static int __dev_queue_xmit(struct sk_buff *skb, struct net_device *sb_dev)
4119 {
4120 	struct net_device *dev = skb->dev;
4121 	struct netdev_queue *txq;
4122 	struct Qdisc *q;
4123 	int rc = -ENOMEM;
4124 	bool again = false;
4125 
4126 	skb_reset_mac_header(skb);
4127 
4128 	if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_SCHED_TSTAMP))
4129 		__skb_tstamp_tx(skb, NULL, NULL, skb->sk, SCM_TSTAMP_SCHED);
4130 
4131 	/* Disable soft irqs for various locks below. Also
4132 	 * stops preemption for RCU.
4133 	 */
4134 	rcu_read_lock_bh();
4135 
4136 	skb_update_prio(skb);
4137 
4138 	qdisc_pkt_len_init(skb);
4139 #ifdef CONFIG_NET_CLS_ACT
4140 	skb->tc_at_ingress = 0;
4141 # ifdef CONFIG_NET_EGRESS
4142 	if (static_branch_unlikely(&egress_needed_key)) {
4143 		skb = sch_handle_egress(skb, &rc, dev);
4144 		if (!skb)
4145 			goto out;
4146 	}
4147 # endif
4148 #endif
4149 	/* If device/qdisc don't need skb->dst, release it right now while
4150 	 * its hot in this cpu cache.
4151 	 */
4152 	if (dev->priv_flags & IFF_XMIT_DST_RELEASE)
4153 		skb_dst_drop(skb);
4154 	else
4155 		skb_dst_force(skb);
4156 
4157 	txq = netdev_core_pick_tx(dev, skb, sb_dev);
4158 	q = rcu_dereference_bh(txq->qdisc);
4159 
4160 	trace_net_dev_queue(skb);
4161 	if (q->enqueue) {
4162 		rc = __dev_xmit_skb(skb, q, dev, txq);
4163 		goto out;
4164 	}
4165 
4166 	/* The device has no queue. Common case for software devices:
4167 	 * loopback, all the sorts of tunnels...
4168 
4169 	 * Really, it is unlikely that netif_tx_lock protection is necessary
4170 	 * here.  (f.e. loopback and IP tunnels are clean ignoring statistics
4171 	 * counters.)
4172 	 * However, it is possible, that they rely on protection
4173 	 * made by us here.
4174 
4175 	 * Check this and shot the lock. It is not prone from deadlocks.
4176 	 *Either shot noqueue qdisc, it is even simpler 8)
4177 	 */
4178 	if (dev->flags & IFF_UP) {
4179 		int cpu = smp_processor_id(); /* ok because BHs are off */
4180 
4181 		if (txq->xmit_lock_owner != cpu) {
4182 			if (dev_xmit_recursion())
4183 				goto recursion_alert;
4184 
4185 			skb = validate_xmit_skb(skb, dev, &again);
4186 			if (!skb)
4187 				goto out;
4188 
4189 			PRANDOM_ADD_NOISE(skb, dev, txq, jiffies);
4190 			HARD_TX_LOCK(dev, txq, cpu);
4191 
4192 			if (!netif_xmit_stopped(txq)) {
4193 				dev_xmit_recursion_inc();
4194 				skb = dev_hard_start_xmit(skb, dev, txq, &rc);
4195 				dev_xmit_recursion_dec();
4196 				if (dev_xmit_complete(rc)) {
4197 					HARD_TX_UNLOCK(dev, txq);
4198 					goto out;
4199 				}
4200 			}
4201 			HARD_TX_UNLOCK(dev, txq);
4202 			net_crit_ratelimited("Virtual device %s asks to queue packet!\n",
4203 					     dev->name);
4204 		} else {
4205 			/* Recursion is detected! It is possible,
4206 			 * unfortunately
4207 			 */
4208 recursion_alert:
4209 			net_crit_ratelimited("Dead loop on virtual device %s, fix it urgently!\n",
4210 					     dev->name);
4211 		}
4212 	}
4213 
4214 	rc = -ENETDOWN;
4215 	rcu_read_unlock_bh();
4216 
4217 	atomic_long_inc(&dev->tx_dropped);
4218 	kfree_skb_list(skb);
4219 	return rc;
4220 out:
4221 	rcu_read_unlock_bh();
4222 	return rc;
4223 }
4224 
4225 int dev_queue_xmit(struct sk_buff *skb)
4226 {
4227 	return __dev_queue_xmit(skb, NULL);
4228 }
4229 EXPORT_SYMBOL(dev_queue_xmit);
4230 
4231 int dev_queue_xmit_accel(struct sk_buff *skb, struct net_device *sb_dev)
4232 {
4233 	return __dev_queue_xmit(skb, sb_dev);
4234 }
4235 EXPORT_SYMBOL(dev_queue_xmit_accel);
4236 
4237 int __dev_direct_xmit(struct sk_buff *skb, u16 queue_id)
4238 {
4239 	struct net_device *dev = skb->dev;
4240 	struct sk_buff *orig_skb = skb;
4241 	struct netdev_queue *txq;
4242 	int ret = NETDEV_TX_BUSY;
4243 	bool again = false;
4244 
4245 	if (unlikely(!netif_running(dev) ||
4246 		     !netif_carrier_ok(dev)))
4247 		goto drop;
4248 
4249 	skb = validate_xmit_skb_list(skb, dev, &again);
4250 	if (skb != orig_skb)
4251 		goto drop;
4252 
4253 	skb_set_queue_mapping(skb, queue_id);
4254 	txq = skb_get_tx_queue(dev, skb);
4255 	PRANDOM_ADD_NOISE(skb, dev, txq, jiffies);
4256 
4257 	local_bh_disable();
4258 
4259 	dev_xmit_recursion_inc();
4260 	HARD_TX_LOCK(dev, txq, smp_processor_id());
4261 	if (!netif_xmit_frozen_or_drv_stopped(txq))
4262 		ret = netdev_start_xmit(skb, dev, txq, false);
4263 	HARD_TX_UNLOCK(dev, txq);
4264 	dev_xmit_recursion_dec();
4265 
4266 	local_bh_enable();
4267 	return ret;
4268 drop:
4269 	atomic_long_inc(&dev->tx_dropped);
4270 	kfree_skb_list(skb);
4271 	return NET_XMIT_DROP;
4272 }
4273 EXPORT_SYMBOL(__dev_direct_xmit);
4274 
4275 /*************************************************************************
4276  *			Receiver routines
4277  *************************************************************************/
4278 
4279 int netdev_max_backlog __read_mostly = 1000;
4280 EXPORT_SYMBOL(netdev_max_backlog);
4281 
4282 int netdev_tstamp_prequeue __read_mostly = 1;
4283 int netdev_budget __read_mostly = 300;
4284 /* Must be at least 2 jiffes to guarantee 1 jiffy timeout */
4285 unsigned int __read_mostly netdev_budget_usecs = 2 * USEC_PER_SEC / HZ;
4286 int weight_p __read_mostly = 64;           /* old backlog weight */
4287 int dev_weight_rx_bias __read_mostly = 1;  /* bias for backlog weight */
4288 int dev_weight_tx_bias __read_mostly = 1;  /* bias for output_queue quota */
4289 int dev_rx_weight __read_mostly = 64;
4290 int dev_tx_weight __read_mostly = 64;
4291 /* Maximum number of GRO_NORMAL skbs to batch up for list-RX */
4292 int gro_normal_batch __read_mostly = 8;
4293 
4294 /* Called with irq disabled */
4295 static inline void ____napi_schedule(struct softnet_data *sd,
4296 				     struct napi_struct *napi)
4297 {
4298 	struct task_struct *thread;
4299 
4300 	if (test_bit(NAPI_STATE_THREADED, &napi->state)) {
4301 		/* Paired with smp_mb__before_atomic() in
4302 		 * napi_enable()/dev_set_threaded().
4303 		 * Use READ_ONCE() to guarantee a complete
4304 		 * read on napi->thread. Only call
4305 		 * wake_up_process() when it's not NULL.
4306 		 */
4307 		thread = READ_ONCE(napi->thread);
4308 		if (thread) {
4309 			/* Avoid doing set_bit() if the thread is in
4310 			 * INTERRUPTIBLE state, cause napi_thread_wait()
4311 			 * makes sure to proceed with napi polling
4312 			 * if the thread is explicitly woken from here.
4313 			 */
4314 			if (READ_ONCE(thread->state) != TASK_INTERRUPTIBLE)
4315 				set_bit(NAPI_STATE_SCHED_THREADED, &napi->state);
4316 			wake_up_process(thread);
4317 			return;
4318 		}
4319 	}
4320 
4321 	list_add_tail(&napi->poll_list, &sd->poll_list);
4322 	__raise_softirq_irqoff(NET_RX_SOFTIRQ);
4323 }
4324 
4325 #ifdef CONFIG_RPS
4326 
4327 /* One global table that all flow-based protocols share. */
4328 struct rps_sock_flow_table __rcu *rps_sock_flow_table __read_mostly;
4329 EXPORT_SYMBOL(rps_sock_flow_table);
4330 u32 rps_cpu_mask __read_mostly;
4331 EXPORT_SYMBOL(rps_cpu_mask);
4332 
4333 struct static_key_false rps_needed __read_mostly;
4334 EXPORT_SYMBOL(rps_needed);
4335 struct static_key_false rfs_needed __read_mostly;
4336 EXPORT_SYMBOL(rfs_needed);
4337 
4338 static struct rps_dev_flow *
4339 set_rps_cpu(struct net_device *dev, struct sk_buff *skb,
4340 	    struct rps_dev_flow *rflow, u16 next_cpu)
4341 {
4342 	if (next_cpu < nr_cpu_ids) {
4343 #ifdef CONFIG_RFS_ACCEL
4344 		struct netdev_rx_queue *rxqueue;
4345 		struct rps_dev_flow_table *flow_table;
4346 		struct rps_dev_flow *old_rflow;
4347 		u32 flow_id;
4348 		u16 rxq_index;
4349 		int rc;
4350 
4351 		/* Should we steer this flow to a different hardware queue? */
4352 		if (!skb_rx_queue_recorded(skb) || !dev->rx_cpu_rmap ||
4353 		    !(dev->features & NETIF_F_NTUPLE))
4354 			goto out;
4355 		rxq_index = cpu_rmap_lookup_index(dev->rx_cpu_rmap, next_cpu);
4356 		if (rxq_index == skb_get_rx_queue(skb))
4357 			goto out;
4358 
4359 		rxqueue = dev->_rx + rxq_index;
4360 		flow_table = rcu_dereference(rxqueue->rps_flow_table);
4361 		if (!flow_table)
4362 			goto out;
4363 		flow_id = skb_get_hash(skb) & flow_table->mask;
4364 		rc = dev->netdev_ops->ndo_rx_flow_steer(dev, skb,
4365 							rxq_index, flow_id);
4366 		if (rc < 0)
4367 			goto out;
4368 		old_rflow = rflow;
4369 		rflow = &flow_table->flows[flow_id];
4370 		rflow->filter = rc;
4371 		if (old_rflow->filter == rflow->filter)
4372 			old_rflow->filter = RPS_NO_FILTER;
4373 	out:
4374 #endif
4375 		rflow->last_qtail =
4376 			per_cpu(softnet_data, next_cpu).input_queue_head;
4377 	}
4378 
4379 	rflow->cpu = next_cpu;
4380 	return rflow;
4381 }
4382 
4383 /*
4384  * get_rps_cpu is called from netif_receive_skb and returns the target
4385  * CPU from the RPS map of the receiving queue for a given skb.
4386  * rcu_read_lock must be held on entry.
4387  */
4388 static int get_rps_cpu(struct net_device *dev, struct sk_buff *skb,
4389 		       struct rps_dev_flow **rflowp)
4390 {
4391 	const struct rps_sock_flow_table *sock_flow_table;
4392 	struct netdev_rx_queue *rxqueue = dev->_rx;
4393 	struct rps_dev_flow_table *flow_table;
4394 	struct rps_map *map;
4395 	int cpu = -1;
4396 	u32 tcpu;
4397 	u32 hash;
4398 
4399 	if (skb_rx_queue_recorded(skb)) {
4400 		u16 index = skb_get_rx_queue(skb);
4401 
4402 		if (unlikely(index >= dev->real_num_rx_queues)) {
4403 			WARN_ONCE(dev->real_num_rx_queues > 1,
4404 				  "%s received packet on queue %u, but number "
4405 				  "of RX queues is %u\n",
4406 				  dev->name, index, dev->real_num_rx_queues);
4407 			goto done;
4408 		}
4409 		rxqueue += index;
4410 	}
4411 
4412 	/* Avoid computing hash if RFS/RPS is not active for this rxqueue */
4413 
4414 	flow_table = rcu_dereference(rxqueue->rps_flow_table);
4415 	map = rcu_dereference(rxqueue->rps_map);
4416 	if (!flow_table && !map)
4417 		goto done;
4418 
4419 	skb_reset_network_header(skb);
4420 	hash = skb_get_hash(skb);
4421 	if (!hash)
4422 		goto done;
4423 
4424 	sock_flow_table = rcu_dereference(rps_sock_flow_table);
4425 	if (flow_table && sock_flow_table) {
4426 		struct rps_dev_flow *rflow;
4427 		u32 next_cpu;
4428 		u32 ident;
4429 
4430 		/* First check into global flow table if there is a match */
4431 		ident = sock_flow_table->ents[hash & sock_flow_table->mask];
4432 		if ((ident ^ hash) & ~rps_cpu_mask)
4433 			goto try_rps;
4434 
4435 		next_cpu = ident & rps_cpu_mask;
4436 
4437 		/* OK, now we know there is a match,
4438 		 * we can look at the local (per receive queue) flow table
4439 		 */
4440 		rflow = &flow_table->flows[hash & flow_table->mask];
4441 		tcpu = rflow->cpu;
4442 
4443 		/*
4444 		 * If the desired CPU (where last recvmsg was done) is
4445 		 * different from current CPU (one in the rx-queue flow
4446 		 * table entry), switch if one of the following holds:
4447 		 *   - Current CPU is unset (>= nr_cpu_ids).
4448 		 *   - Current CPU is offline.
4449 		 *   - The current CPU's queue tail has advanced beyond the
4450 		 *     last packet that was enqueued using this table entry.
4451 		 *     This guarantees that all previous packets for the flow
4452 		 *     have been dequeued, thus preserving in order delivery.
4453 		 */
4454 		if (unlikely(tcpu != next_cpu) &&
4455 		    (tcpu >= nr_cpu_ids || !cpu_online(tcpu) ||
4456 		     ((int)(per_cpu(softnet_data, tcpu).input_queue_head -
4457 		      rflow->last_qtail)) >= 0)) {
4458 			tcpu = next_cpu;
4459 			rflow = set_rps_cpu(dev, skb, rflow, next_cpu);
4460 		}
4461 
4462 		if (tcpu < nr_cpu_ids && cpu_online(tcpu)) {
4463 			*rflowp = rflow;
4464 			cpu = tcpu;
4465 			goto done;
4466 		}
4467 	}
4468 
4469 try_rps:
4470 
4471 	if (map) {
4472 		tcpu = map->cpus[reciprocal_scale(hash, map->len)];
4473 		if (cpu_online(tcpu)) {
4474 			cpu = tcpu;
4475 			goto done;
4476 		}
4477 	}
4478 
4479 done:
4480 	return cpu;
4481 }
4482 
4483 #ifdef CONFIG_RFS_ACCEL
4484 
4485 /**
4486  * rps_may_expire_flow - check whether an RFS hardware filter may be removed
4487  * @dev: Device on which the filter was set
4488  * @rxq_index: RX queue index
4489  * @flow_id: Flow ID passed to ndo_rx_flow_steer()
4490  * @filter_id: Filter ID returned by ndo_rx_flow_steer()
4491  *
4492  * Drivers that implement ndo_rx_flow_steer() should periodically call
4493  * this function for each installed filter and remove the filters for
4494  * which it returns %true.
4495  */
4496 bool rps_may_expire_flow(struct net_device *dev, u16 rxq_index,
4497 			 u32 flow_id, u16 filter_id)
4498 {
4499 	struct netdev_rx_queue *rxqueue = dev->_rx + rxq_index;
4500 	struct rps_dev_flow_table *flow_table;
4501 	struct rps_dev_flow *rflow;
4502 	bool expire = true;
4503 	unsigned int cpu;
4504 
4505 	rcu_read_lock();
4506 	flow_table = rcu_dereference(rxqueue->rps_flow_table);
4507 	if (flow_table && flow_id <= flow_table->mask) {
4508 		rflow = &flow_table->flows[flow_id];
4509 		cpu = READ_ONCE(rflow->cpu);
4510 		if (rflow->filter == filter_id && cpu < nr_cpu_ids &&
4511 		    ((int)(per_cpu(softnet_data, cpu).input_queue_head -
4512 			   rflow->last_qtail) <
4513 		     (int)(10 * flow_table->mask)))
4514 			expire = false;
4515 	}
4516 	rcu_read_unlock();
4517 	return expire;
4518 }
4519 EXPORT_SYMBOL(rps_may_expire_flow);
4520 
4521 #endif /* CONFIG_RFS_ACCEL */
4522 
4523 /* Called from hardirq (IPI) context */
4524 static void rps_trigger_softirq(void *data)
4525 {
4526 	struct softnet_data *sd = data;
4527 
4528 	____napi_schedule(sd, &sd->backlog);
4529 	sd->received_rps++;
4530 }
4531 
4532 #endif /* CONFIG_RPS */
4533 
4534 /*
4535  * Check if this softnet_data structure is another cpu one
4536  * If yes, queue it to our IPI list and return 1
4537  * If no, return 0
4538  */
4539 static int rps_ipi_queued(struct softnet_data *sd)
4540 {
4541 #ifdef CONFIG_RPS
4542 	struct softnet_data *mysd = this_cpu_ptr(&softnet_data);
4543 
4544 	if (sd != mysd) {
4545 		sd->rps_ipi_next = mysd->rps_ipi_list;
4546 		mysd->rps_ipi_list = sd;
4547 
4548 		__raise_softirq_irqoff(NET_RX_SOFTIRQ);
4549 		return 1;
4550 	}
4551 #endif /* CONFIG_RPS */
4552 	return 0;
4553 }
4554 
4555 #ifdef CONFIG_NET_FLOW_LIMIT
4556 int netdev_flow_limit_table_len __read_mostly = (1 << 12);
4557 #endif
4558 
4559 static bool skb_flow_limit(struct sk_buff *skb, unsigned int qlen)
4560 {
4561 #ifdef CONFIG_NET_FLOW_LIMIT
4562 	struct sd_flow_limit *fl;
4563 	struct softnet_data *sd;
4564 	unsigned int old_flow, new_flow;
4565 
4566 	if (qlen < (netdev_max_backlog >> 1))
4567 		return false;
4568 
4569 	sd = this_cpu_ptr(&softnet_data);
4570 
4571 	rcu_read_lock();
4572 	fl = rcu_dereference(sd->flow_limit);
4573 	if (fl) {
4574 		new_flow = skb_get_hash(skb) & (fl->num_buckets - 1);
4575 		old_flow = fl->history[fl->history_head];
4576 		fl->history[fl->history_head] = new_flow;
4577 
4578 		fl->history_head++;
4579 		fl->history_head &= FLOW_LIMIT_HISTORY - 1;
4580 
4581 		if (likely(fl->buckets[old_flow]))
4582 			fl->buckets[old_flow]--;
4583 
4584 		if (++fl->buckets[new_flow] > (FLOW_LIMIT_HISTORY >> 1)) {
4585 			fl->count++;
4586 			rcu_read_unlock();
4587 			return true;
4588 		}
4589 	}
4590 	rcu_read_unlock();
4591 #endif
4592 	return false;
4593 }
4594 
4595 /*
4596  * enqueue_to_backlog is called to queue an skb to a per CPU backlog
4597  * queue (may be a remote CPU queue).
4598  */
4599 static int enqueue_to_backlog(struct sk_buff *skb, int cpu,
4600 			      unsigned int *qtail)
4601 {
4602 	struct softnet_data *sd;
4603 	unsigned long flags;
4604 	unsigned int qlen;
4605 
4606 	sd = &per_cpu(softnet_data, cpu);
4607 
4608 	local_irq_save(flags);
4609 
4610 	rps_lock(sd);
4611 	if (!netif_running(skb->dev))
4612 		goto drop;
4613 	qlen = skb_queue_len(&sd->input_pkt_queue);
4614 	if (qlen <= netdev_max_backlog && !skb_flow_limit(skb, qlen)) {
4615 		if (qlen) {
4616 enqueue:
4617 			__skb_queue_tail(&sd->input_pkt_queue, skb);
4618 			input_queue_tail_incr_save(sd, qtail);
4619 			rps_unlock(sd);
4620 			local_irq_restore(flags);
4621 			return NET_RX_SUCCESS;
4622 		}
4623 
4624 		/* Schedule NAPI for backlog device
4625 		 * We can use non atomic operation since we own the queue lock
4626 		 */
4627 		if (!__test_and_set_bit(NAPI_STATE_SCHED, &sd->backlog.state)) {
4628 			if (!rps_ipi_queued(sd))
4629 				____napi_schedule(sd, &sd->backlog);
4630 		}
4631 		goto enqueue;
4632 	}
4633 
4634 drop:
4635 	sd->dropped++;
4636 	rps_unlock(sd);
4637 
4638 	local_irq_restore(flags);
4639 
4640 	atomic_long_inc(&skb->dev->rx_dropped);
4641 	kfree_skb(skb);
4642 	return NET_RX_DROP;
4643 }
4644 
4645 static struct netdev_rx_queue *netif_get_rxqueue(struct sk_buff *skb)
4646 {
4647 	struct net_device *dev = skb->dev;
4648 	struct netdev_rx_queue *rxqueue;
4649 
4650 	rxqueue = dev->_rx;
4651 
4652 	if (skb_rx_queue_recorded(skb)) {
4653 		u16 index = skb_get_rx_queue(skb);
4654 
4655 		if (unlikely(index >= dev->real_num_rx_queues)) {
4656 			WARN_ONCE(dev->real_num_rx_queues > 1,
4657 				  "%s received packet on queue %u, but number "
4658 				  "of RX queues is %u\n",
4659 				  dev->name, index, dev->real_num_rx_queues);
4660 
4661 			return rxqueue; /* Return first rxqueue */
4662 		}
4663 		rxqueue += index;
4664 	}
4665 	return rxqueue;
4666 }
4667 
4668 static u32 netif_receive_generic_xdp(struct sk_buff *skb,
4669 				     struct xdp_buff *xdp,
4670 				     struct bpf_prog *xdp_prog)
4671 {
4672 	void *orig_data, *orig_data_end, *hard_start;
4673 	struct netdev_rx_queue *rxqueue;
4674 	u32 metalen, act = XDP_DROP;
4675 	u32 mac_len, frame_sz;
4676 	__be16 orig_eth_type;
4677 	struct ethhdr *eth;
4678 	bool orig_bcast;
4679 	int off;
4680 
4681 	/* Reinjected packets coming from act_mirred or similar should
4682 	 * not get XDP generic processing.
4683 	 */
4684 	if (skb_is_redirected(skb))
4685 		return XDP_PASS;
4686 
4687 	/* XDP packets must be linear and must have sufficient headroom
4688 	 * of XDP_PACKET_HEADROOM bytes. This is the guarantee that also
4689 	 * native XDP provides, thus we need to do it here as well.
4690 	 */
4691 	if (skb_cloned(skb) || skb_is_nonlinear(skb) ||
4692 	    skb_headroom(skb) < XDP_PACKET_HEADROOM) {
4693 		int hroom = XDP_PACKET_HEADROOM - skb_headroom(skb);
4694 		int troom = skb->tail + skb->data_len - skb->end;
4695 
4696 		/* In case we have to go down the path and also linearize,
4697 		 * then lets do the pskb_expand_head() work just once here.
4698 		 */
4699 		if (pskb_expand_head(skb,
4700 				     hroom > 0 ? ALIGN(hroom, NET_SKB_PAD) : 0,
4701 				     troom > 0 ? troom + 128 : 0, GFP_ATOMIC))
4702 			goto do_drop;
4703 		if (skb_linearize(skb))
4704 			goto do_drop;
4705 	}
4706 
4707 	/* The XDP program wants to see the packet starting at the MAC
4708 	 * header.
4709 	 */
4710 	mac_len = skb->data - skb_mac_header(skb);
4711 	hard_start = skb->data - skb_headroom(skb);
4712 
4713 	/* SKB "head" area always have tailroom for skb_shared_info */
4714 	frame_sz = (void *)skb_end_pointer(skb) - hard_start;
4715 	frame_sz += SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
4716 
4717 	rxqueue = netif_get_rxqueue(skb);
4718 	xdp_init_buff(xdp, frame_sz, &rxqueue->xdp_rxq);
4719 	xdp_prepare_buff(xdp, hard_start, skb_headroom(skb) - mac_len,
4720 			 skb_headlen(skb) + mac_len, true);
4721 
4722 	orig_data_end = xdp->data_end;
4723 	orig_data = xdp->data;
4724 	eth = (struct ethhdr *)xdp->data;
4725 	orig_bcast = is_multicast_ether_addr_64bits(eth->h_dest);
4726 	orig_eth_type = eth->h_proto;
4727 
4728 	act = bpf_prog_run_xdp(xdp_prog, xdp);
4729 
4730 	/* check if bpf_xdp_adjust_head was used */
4731 	off = xdp->data - orig_data;
4732 	if (off) {
4733 		if (off > 0)
4734 			__skb_pull(skb, off);
4735 		else if (off < 0)
4736 			__skb_push(skb, -off);
4737 
4738 		skb->mac_header += off;
4739 		skb_reset_network_header(skb);
4740 	}
4741 
4742 	/* check if bpf_xdp_adjust_tail was used */
4743 	off = xdp->data_end - orig_data_end;
4744 	if (off != 0) {
4745 		skb_set_tail_pointer(skb, xdp->data_end - xdp->data);
4746 		skb->len += off; /* positive on grow, negative on shrink */
4747 	}
4748 
4749 	/* check if XDP changed eth hdr such SKB needs update */
4750 	eth = (struct ethhdr *)xdp->data;
4751 	if ((orig_eth_type != eth->h_proto) ||
4752 	    (orig_bcast != is_multicast_ether_addr_64bits(eth->h_dest))) {
4753 		__skb_push(skb, ETH_HLEN);
4754 		skb->protocol = eth_type_trans(skb, skb->dev);
4755 	}
4756 
4757 	switch (act) {
4758 	case XDP_REDIRECT:
4759 	case XDP_TX:
4760 		__skb_push(skb, mac_len);
4761 		break;
4762 	case XDP_PASS:
4763 		metalen = xdp->data - xdp->data_meta;
4764 		if (metalen)
4765 			skb_metadata_set(skb, metalen);
4766 		break;
4767 	default:
4768 		bpf_warn_invalid_xdp_action(act);
4769 		fallthrough;
4770 	case XDP_ABORTED:
4771 		trace_xdp_exception(skb->dev, xdp_prog, act);
4772 		fallthrough;
4773 	case XDP_DROP:
4774 	do_drop:
4775 		kfree_skb(skb);
4776 		break;
4777 	}
4778 
4779 	return act;
4780 }
4781 
4782 /* When doing generic XDP we have to bypass the qdisc layer and the
4783  * network taps in order to match in-driver-XDP behavior.
4784  */
4785 void generic_xdp_tx(struct sk_buff *skb, struct bpf_prog *xdp_prog)
4786 {
4787 	struct net_device *dev = skb->dev;
4788 	struct netdev_queue *txq;
4789 	bool free_skb = true;
4790 	int cpu, rc;
4791 
4792 	txq = netdev_core_pick_tx(dev, skb, NULL);
4793 	cpu = smp_processor_id();
4794 	HARD_TX_LOCK(dev, txq, cpu);
4795 	if (!netif_xmit_stopped(txq)) {
4796 		rc = netdev_start_xmit(skb, dev, txq, 0);
4797 		if (dev_xmit_complete(rc))
4798 			free_skb = false;
4799 	}
4800 	HARD_TX_UNLOCK(dev, txq);
4801 	if (free_skb) {
4802 		trace_xdp_exception(dev, xdp_prog, XDP_TX);
4803 		kfree_skb(skb);
4804 	}
4805 }
4806 
4807 static DEFINE_STATIC_KEY_FALSE(generic_xdp_needed_key);
4808 
4809 int do_xdp_generic(struct bpf_prog *xdp_prog, struct sk_buff *skb)
4810 {
4811 	if (xdp_prog) {
4812 		struct xdp_buff xdp;
4813 		u32 act;
4814 		int err;
4815 
4816 		act = netif_receive_generic_xdp(skb, &xdp, xdp_prog);
4817 		if (act != XDP_PASS) {
4818 			switch (act) {
4819 			case XDP_REDIRECT:
4820 				err = xdp_do_generic_redirect(skb->dev, skb,
4821 							      &xdp, xdp_prog);
4822 				if (err)
4823 					goto out_redir;
4824 				break;
4825 			case XDP_TX:
4826 				generic_xdp_tx(skb, xdp_prog);
4827 				break;
4828 			}
4829 			return XDP_DROP;
4830 		}
4831 	}
4832 	return XDP_PASS;
4833 out_redir:
4834 	kfree_skb(skb);
4835 	return XDP_DROP;
4836 }
4837 EXPORT_SYMBOL_GPL(do_xdp_generic);
4838 
4839 static int netif_rx_internal(struct sk_buff *skb)
4840 {
4841 	int ret;
4842 
4843 	net_timestamp_check(netdev_tstamp_prequeue, skb);
4844 
4845 	trace_netif_rx(skb);
4846 
4847 #ifdef CONFIG_RPS
4848 	if (static_branch_unlikely(&rps_needed)) {
4849 		struct rps_dev_flow voidflow, *rflow = &voidflow;
4850 		int cpu;
4851 
4852 		preempt_disable();
4853 		rcu_read_lock();
4854 
4855 		cpu = get_rps_cpu(skb->dev, skb, &rflow);
4856 		if (cpu < 0)
4857 			cpu = smp_processor_id();
4858 
4859 		ret = enqueue_to_backlog(skb, cpu, &rflow->last_qtail);
4860 
4861 		rcu_read_unlock();
4862 		preempt_enable();
4863 	} else
4864 #endif
4865 	{
4866 		unsigned int qtail;
4867 
4868 		ret = enqueue_to_backlog(skb, get_cpu(), &qtail);
4869 		put_cpu();
4870 	}
4871 	return ret;
4872 }
4873 
4874 /**
4875  *	netif_rx	-	post buffer to the network code
4876  *	@skb: buffer to post
4877  *
4878  *	This function receives a packet from a device driver and queues it for
4879  *	the upper (protocol) levels to process.  It always succeeds. The buffer
4880  *	may be dropped during processing for congestion control or by the
4881  *	protocol layers.
4882  *
4883  *	return values:
4884  *	NET_RX_SUCCESS	(no congestion)
4885  *	NET_RX_DROP     (packet was dropped)
4886  *
4887  */
4888 
4889 int netif_rx(struct sk_buff *skb)
4890 {
4891 	int ret;
4892 
4893 	trace_netif_rx_entry(skb);
4894 
4895 	ret = netif_rx_internal(skb);
4896 	trace_netif_rx_exit(ret);
4897 
4898 	return ret;
4899 }
4900 EXPORT_SYMBOL(netif_rx);
4901 
4902 int netif_rx_ni(struct sk_buff *skb)
4903 {
4904 	int err;
4905 
4906 	trace_netif_rx_ni_entry(skb);
4907 
4908 	preempt_disable();
4909 	err = netif_rx_internal(skb);
4910 	if (local_softirq_pending())
4911 		do_softirq();
4912 	preempt_enable();
4913 	trace_netif_rx_ni_exit(err);
4914 
4915 	return err;
4916 }
4917 EXPORT_SYMBOL(netif_rx_ni);
4918 
4919 int netif_rx_any_context(struct sk_buff *skb)
4920 {
4921 	/*
4922 	 * If invoked from contexts which do not invoke bottom half
4923 	 * processing either at return from interrupt or when softrqs are
4924 	 * reenabled, use netif_rx_ni() which invokes bottomhalf processing
4925 	 * directly.
4926 	 */
4927 	if (in_interrupt())
4928 		return netif_rx(skb);
4929 	else
4930 		return netif_rx_ni(skb);
4931 }
4932 EXPORT_SYMBOL(netif_rx_any_context);
4933 
4934 static __latent_entropy void net_tx_action(struct softirq_action *h)
4935 {
4936 	struct softnet_data *sd = this_cpu_ptr(&softnet_data);
4937 
4938 	if (sd->completion_queue) {
4939 		struct sk_buff *clist;
4940 
4941 		local_irq_disable();
4942 		clist = sd->completion_queue;
4943 		sd->completion_queue = NULL;
4944 		local_irq_enable();
4945 
4946 		while (clist) {
4947 			struct sk_buff *skb = clist;
4948 
4949 			clist = clist->next;
4950 
4951 			WARN_ON(refcount_read(&skb->users));
4952 			if (likely(get_kfree_skb_cb(skb)->reason == SKB_REASON_CONSUMED))
4953 				trace_consume_skb(skb);
4954 			else
4955 				trace_kfree_skb(skb, net_tx_action);
4956 
4957 			if (skb->fclone != SKB_FCLONE_UNAVAILABLE)
4958 				__kfree_skb(skb);
4959 			else
4960 				__kfree_skb_defer(skb);
4961 		}
4962 	}
4963 
4964 	if (sd->output_queue) {
4965 		struct Qdisc *head;
4966 
4967 		local_irq_disable();
4968 		head = sd->output_queue;
4969 		sd->output_queue = NULL;
4970 		sd->output_queue_tailp = &sd->output_queue;
4971 		local_irq_enable();
4972 
4973 		while (head) {
4974 			struct Qdisc *q = head;
4975 			spinlock_t *root_lock = NULL;
4976 
4977 			head = head->next_sched;
4978 
4979 			if (!(q->flags & TCQ_F_NOLOCK)) {
4980 				root_lock = qdisc_lock(q);
4981 				spin_lock(root_lock);
4982 			}
4983 			/* We need to make sure head->next_sched is read
4984 			 * before clearing __QDISC_STATE_SCHED
4985 			 */
4986 			smp_mb__before_atomic();
4987 			clear_bit(__QDISC_STATE_SCHED, &q->state);
4988 			qdisc_run(q);
4989 			if (root_lock)
4990 				spin_unlock(root_lock);
4991 		}
4992 	}
4993 
4994 	xfrm_dev_backlog(sd);
4995 }
4996 
4997 #if IS_ENABLED(CONFIG_BRIDGE) && IS_ENABLED(CONFIG_ATM_LANE)
4998 /* This hook is defined here for ATM LANE */
4999 int (*br_fdb_test_addr_hook)(struct net_device *dev,
5000 			     unsigned char *addr) __read_mostly;
5001 EXPORT_SYMBOL_GPL(br_fdb_test_addr_hook);
5002 #endif
5003 
5004 static inline struct sk_buff *
5005 sch_handle_ingress(struct sk_buff *skb, struct packet_type **pt_prev, int *ret,
5006 		   struct net_device *orig_dev, bool *another)
5007 {
5008 #ifdef CONFIG_NET_CLS_ACT
5009 	struct mini_Qdisc *miniq = rcu_dereference_bh(skb->dev->miniq_ingress);
5010 	struct tcf_result cl_res;
5011 
5012 	/* If there's at least one ingress present somewhere (so
5013 	 * we get here via enabled static key), remaining devices
5014 	 * that are not configured with an ingress qdisc will bail
5015 	 * out here.
5016 	 */
5017 	if (!miniq)
5018 		return skb;
5019 
5020 	if (*pt_prev) {
5021 		*ret = deliver_skb(skb, *pt_prev, orig_dev);
5022 		*pt_prev = NULL;
5023 	}
5024 
5025 	qdisc_skb_cb(skb)->pkt_len = skb->len;
5026 	qdisc_skb_cb(skb)->mru = 0;
5027 	qdisc_skb_cb(skb)->post_ct = false;
5028 	skb->tc_at_ingress = 1;
5029 	mini_qdisc_bstats_cpu_update(miniq, skb);
5030 
5031 	switch (tcf_classify_ingress(skb, miniq->block, miniq->filter_list,
5032 				     &cl_res, false)) {
5033 	case TC_ACT_OK:
5034 	case TC_ACT_RECLASSIFY:
5035 		skb->tc_index = TC_H_MIN(cl_res.classid);
5036 		break;
5037 	case TC_ACT_SHOT:
5038 		mini_qdisc_qstats_cpu_drop(miniq);
5039 		kfree_skb(skb);
5040 		return NULL;
5041 	case TC_ACT_STOLEN:
5042 	case TC_ACT_QUEUED:
5043 	case TC_ACT_TRAP:
5044 		consume_skb(skb);
5045 		return NULL;
5046 	case TC_ACT_REDIRECT:
5047 		/* skb_mac_header check was done by cls/act_bpf, so
5048 		 * we can safely push the L2 header back before
5049 		 * redirecting to another netdev
5050 		 */
5051 		__skb_push(skb, skb->mac_len);
5052 		if (skb_do_redirect(skb) == -EAGAIN) {
5053 			__skb_pull(skb, skb->mac_len);
5054 			*another = true;
5055 			break;
5056 		}
5057 		return NULL;
5058 	case TC_ACT_CONSUMED:
5059 		return NULL;
5060 	default:
5061 		break;
5062 	}
5063 #endif /* CONFIG_NET_CLS_ACT */
5064 	return skb;
5065 }
5066 
5067 /**
5068  *	netdev_is_rx_handler_busy - check if receive handler is registered
5069  *	@dev: device to check
5070  *
5071  *	Check if a receive handler is already registered for a given device.
5072  *	Return true if there one.
5073  *
5074  *	The caller must hold the rtnl_mutex.
5075  */
5076 bool netdev_is_rx_handler_busy(struct net_device *dev)
5077 {
5078 	ASSERT_RTNL();
5079 	return dev && rtnl_dereference(dev->rx_handler);
5080 }
5081 EXPORT_SYMBOL_GPL(netdev_is_rx_handler_busy);
5082 
5083 /**
5084  *	netdev_rx_handler_register - register receive handler
5085  *	@dev: device to register a handler for
5086  *	@rx_handler: receive handler to register
5087  *	@rx_handler_data: data pointer that is used by rx handler
5088  *
5089  *	Register a receive handler for a device. This handler will then be
5090  *	called from __netif_receive_skb. A negative errno code is returned
5091  *	on a failure.
5092  *
5093  *	The caller must hold the rtnl_mutex.
5094  *
5095  *	For a general description of rx_handler, see enum rx_handler_result.
5096  */
5097 int netdev_rx_handler_register(struct net_device *dev,
5098 			       rx_handler_func_t *rx_handler,
5099 			       void *rx_handler_data)
5100 {
5101 	if (netdev_is_rx_handler_busy(dev))
5102 		return -EBUSY;
5103 
5104 	if (dev->priv_flags & IFF_NO_RX_HANDLER)
5105 		return -EINVAL;
5106 
5107 	/* Note: rx_handler_data must be set before rx_handler */
5108 	rcu_assign_pointer(dev->rx_handler_data, rx_handler_data);
5109 	rcu_assign_pointer(dev->rx_handler, rx_handler);
5110 
5111 	return 0;
5112 }
5113 EXPORT_SYMBOL_GPL(netdev_rx_handler_register);
5114 
5115 /**
5116  *	netdev_rx_handler_unregister - unregister receive handler
5117  *	@dev: device to unregister a handler from
5118  *
5119  *	Unregister a receive handler from a device.
5120  *
5121  *	The caller must hold the rtnl_mutex.
5122  */
5123 void netdev_rx_handler_unregister(struct net_device *dev)
5124 {
5125 
5126 	ASSERT_RTNL();
5127 	RCU_INIT_POINTER(dev->rx_handler, NULL);
5128 	/* a reader seeing a non NULL rx_handler in a rcu_read_lock()
5129 	 * section has a guarantee to see a non NULL rx_handler_data
5130 	 * as well.
5131 	 */
5132 	synchronize_net();
5133 	RCU_INIT_POINTER(dev->rx_handler_data, NULL);
5134 }
5135 EXPORT_SYMBOL_GPL(netdev_rx_handler_unregister);
5136 
5137 /*
5138  * Limit the use of PFMEMALLOC reserves to those protocols that implement
5139  * the special handling of PFMEMALLOC skbs.
5140  */
5141 static bool skb_pfmemalloc_protocol(struct sk_buff *skb)
5142 {
5143 	switch (skb->protocol) {
5144 	case htons(ETH_P_ARP):
5145 	case htons(ETH_P_IP):
5146 	case htons(ETH_P_IPV6):
5147 	case htons(ETH_P_8021Q):
5148 	case htons(ETH_P_8021AD):
5149 		return true;
5150 	default:
5151 		return false;
5152 	}
5153 }
5154 
5155 static inline int nf_ingress(struct sk_buff *skb, struct packet_type **pt_prev,
5156 			     int *ret, struct net_device *orig_dev)
5157 {
5158 	if (nf_hook_ingress_active(skb)) {
5159 		int ingress_retval;
5160 
5161 		if (*pt_prev) {
5162 			*ret = deliver_skb(skb, *pt_prev, orig_dev);
5163 			*pt_prev = NULL;
5164 		}
5165 
5166 		rcu_read_lock();
5167 		ingress_retval = nf_hook_ingress(skb);
5168 		rcu_read_unlock();
5169 		return ingress_retval;
5170 	}
5171 	return 0;
5172 }
5173 
5174 static int __netif_receive_skb_core(struct sk_buff **pskb, bool pfmemalloc,
5175 				    struct packet_type **ppt_prev)
5176 {
5177 	struct packet_type *ptype, *pt_prev;
5178 	rx_handler_func_t *rx_handler;
5179 	struct sk_buff *skb = *pskb;
5180 	struct net_device *orig_dev;
5181 	bool deliver_exact = false;
5182 	int ret = NET_RX_DROP;
5183 	__be16 type;
5184 
5185 	net_timestamp_check(!netdev_tstamp_prequeue, skb);
5186 
5187 	trace_netif_receive_skb(skb);
5188 
5189 	orig_dev = skb->dev;
5190 
5191 	skb_reset_network_header(skb);
5192 	if (!skb_transport_header_was_set(skb))
5193 		skb_reset_transport_header(skb);
5194 	skb_reset_mac_len(skb);
5195 
5196 	pt_prev = NULL;
5197 
5198 another_round:
5199 	skb->skb_iif = skb->dev->ifindex;
5200 
5201 	__this_cpu_inc(softnet_data.processed);
5202 
5203 	if (static_branch_unlikely(&generic_xdp_needed_key)) {
5204 		int ret2;
5205 
5206 		preempt_disable();
5207 		ret2 = do_xdp_generic(rcu_dereference(skb->dev->xdp_prog), skb);
5208 		preempt_enable();
5209 
5210 		if (ret2 != XDP_PASS) {
5211 			ret = NET_RX_DROP;
5212 			goto out;
5213 		}
5214 		skb_reset_mac_len(skb);
5215 	}
5216 
5217 	if (eth_type_vlan(skb->protocol)) {
5218 		skb = skb_vlan_untag(skb);
5219 		if (unlikely(!skb))
5220 			goto out;
5221 	}
5222 
5223 	if (skb_skip_tc_classify(skb))
5224 		goto skip_classify;
5225 
5226 	if (pfmemalloc)
5227 		goto skip_taps;
5228 
5229 	list_for_each_entry_rcu(ptype, &ptype_all, list) {
5230 		if (pt_prev)
5231 			ret = deliver_skb(skb, pt_prev, orig_dev);
5232 		pt_prev = ptype;
5233 	}
5234 
5235 	list_for_each_entry_rcu(ptype, &skb->dev->ptype_all, list) {
5236 		if (pt_prev)
5237 			ret = deliver_skb(skb, pt_prev, orig_dev);
5238 		pt_prev = ptype;
5239 	}
5240 
5241 skip_taps:
5242 #ifdef CONFIG_NET_INGRESS
5243 	if (static_branch_unlikely(&ingress_needed_key)) {
5244 		bool another = false;
5245 
5246 		skb = sch_handle_ingress(skb, &pt_prev, &ret, orig_dev,
5247 					 &another);
5248 		if (another)
5249 			goto another_round;
5250 		if (!skb)
5251 			goto out;
5252 
5253 		if (nf_ingress(skb, &pt_prev, &ret, orig_dev) < 0)
5254 			goto out;
5255 	}
5256 #endif
5257 	skb_reset_redirect(skb);
5258 skip_classify:
5259 	if (pfmemalloc && !skb_pfmemalloc_protocol(skb))
5260 		goto drop;
5261 
5262 	if (skb_vlan_tag_present(skb)) {
5263 		if (pt_prev) {
5264 			ret = deliver_skb(skb, pt_prev, orig_dev);
5265 			pt_prev = NULL;
5266 		}
5267 		if (vlan_do_receive(&skb))
5268 			goto another_round;
5269 		else if (unlikely(!skb))
5270 			goto out;
5271 	}
5272 
5273 	rx_handler = rcu_dereference(skb->dev->rx_handler);
5274 	if (rx_handler) {
5275 		if (pt_prev) {
5276 			ret = deliver_skb(skb, pt_prev, orig_dev);
5277 			pt_prev = NULL;
5278 		}
5279 		switch (rx_handler(&skb)) {
5280 		case RX_HANDLER_CONSUMED:
5281 			ret = NET_RX_SUCCESS;
5282 			goto out;
5283 		case RX_HANDLER_ANOTHER:
5284 			goto another_round;
5285 		case RX_HANDLER_EXACT:
5286 			deliver_exact = true;
5287 		case RX_HANDLER_PASS:
5288 			break;
5289 		default:
5290 			BUG();
5291 		}
5292 	}
5293 
5294 	if (unlikely(skb_vlan_tag_present(skb)) && !netdev_uses_dsa(skb->dev)) {
5295 check_vlan_id:
5296 		if (skb_vlan_tag_get_id(skb)) {
5297 			/* Vlan id is non 0 and vlan_do_receive() above couldn't
5298 			 * find vlan device.
5299 			 */
5300 			skb->pkt_type = PACKET_OTHERHOST;
5301 		} else if (eth_type_vlan(skb->protocol)) {
5302 			/* Outer header is 802.1P with vlan 0, inner header is
5303 			 * 802.1Q or 802.1AD and vlan_do_receive() above could
5304 			 * not find vlan dev for vlan id 0.
5305 			 */
5306 			__vlan_hwaccel_clear_tag(skb);
5307 			skb = skb_vlan_untag(skb);
5308 			if (unlikely(!skb))
5309 				goto out;
5310 			if (vlan_do_receive(&skb))
5311 				/* After stripping off 802.1P header with vlan 0
5312 				 * vlan dev is found for inner header.
5313 				 */
5314 				goto another_round;
5315 			else if (unlikely(!skb))
5316 				goto out;
5317 			else
5318 				/* We have stripped outer 802.1P vlan 0 header.
5319 				 * But could not find vlan dev.
5320 				 * check again for vlan id to set OTHERHOST.
5321 				 */
5322 				goto check_vlan_id;
5323 		}
5324 		/* Note: we might in the future use prio bits
5325 		 * and set skb->priority like in vlan_do_receive()
5326 		 * For the time being, just ignore Priority Code Point
5327 		 */
5328 		__vlan_hwaccel_clear_tag(skb);
5329 	}
5330 
5331 	type = skb->protocol;
5332 
5333 	/* deliver only exact match when indicated */
5334 	if (likely(!deliver_exact)) {
5335 		deliver_ptype_list_skb(skb, &pt_prev, orig_dev, type,
5336 				       &ptype_base[ntohs(type) &
5337 						   PTYPE_HASH_MASK]);
5338 	}
5339 
5340 	deliver_ptype_list_skb(skb, &pt_prev, orig_dev, type,
5341 			       &orig_dev->ptype_specific);
5342 
5343 	if (unlikely(skb->dev != orig_dev)) {
5344 		deliver_ptype_list_skb(skb, &pt_prev, orig_dev, type,
5345 				       &skb->dev->ptype_specific);
5346 	}
5347 
5348 	if (pt_prev) {
5349 		if (unlikely(skb_orphan_frags_rx(skb, GFP_ATOMIC)))
5350 			goto drop;
5351 		*ppt_prev = pt_prev;
5352 	} else {
5353 drop:
5354 		if (!deliver_exact)
5355 			atomic_long_inc(&skb->dev->rx_dropped);
5356 		else
5357 			atomic_long_inc(&skb->dev->rx_nohandler);
5358 		kfree_skb(skb);
5359 		/* Jamal, now you will not able to escape explaining
5360 		 * me how you were going to use this. :-)
5361 		 */
5362 		ret = NET_RX_DROP;
5363 	}
5364 
5365 out:
5366 	/* The invariant here is that if *ppt_prev is not NULL
5367 	 * then skb should also be non-NULL.
5368 	 *
5369 	 * Apparently *ppt_prev assignment above holds this invariant due to
5370 	 * skb dereferencing near it.
5371 	 */
5372 	*pskb = skb;
5373 	return ret;
5374 }
5375 
5376 static int __netif_receive_skb_one_core(struct sk_buff *skb, bool pfmemalloc)
5377 {
5378 	struct net_device *orig_dev = skb->dev;
5379 	struct packet_type *pt_prev = NULL;
5380 	int ret;
5381 
5382 	ret = __netif_receive_skb_core(&skb, pfmemalloc, &pt_prev);
5383 	if (pt_prev)
5384 		ret = INDIRECT_CALL_INET(pt_prev->func, ipv6_rcv, ip_rcv, skb,
5385 					 skb->dev, pt_prev, orig_dev);
5386 	return ret;
5387 }
5388 
5389 /**
5390  *	netif_receive_skb_core - special purpose version of netif_receive_skb
5391  *	@skb: buffer to process
5392  *
5393  *	More direct receive version of netif_receive_skb().  It should
5394  *	only be used by callers that have a need to skip RPS and Generic XDP.
5395  *	Caller must also take care of handling if ``(page_is_)pfmemalloc``.
5396  *
5397  *	This function may only be called from softirq context and interrupts
5398  *	should be enabled.
5399  *
5400  *	Return values (usually ignored):
5401  *	NET_RX_SUCCESS: no congestion
5402  *	NET_RX_DROP: packet was dropped
5403  */
5404 int netif_receive_skb_core(struct sk_buff *skb)
5405 {
5406 	int ret;
5407 
5408 	rcu_read_lock();
5409 	ret = __netif_receive_skb_one_core(skb, false);
5410 	rcu_read_unlock();
5411 
5412 	return ret;
5413 }
5414 EXPORT_SYMBOL(netif_receive_skb_core);
5415 
5416 static inline void __netif_receive_skb_list_ptype(struct list_head *head,
5417 						  struct packet_type *pt_prev,
5418 						  struct net_device *orig_dev)
5419 {
5420 	struct sk_buff *skb, *next;
5421 
5422 	if (!pt_prev)
5423 		return;
5424 	if (list_empty(head))
5425 		return;
5426 	if (pt_prev->list_func != NULL)
5427 		INDIRECT_CALL_INET(pt_prev->list_func, ipv6_list_rcv,
5428 				   ip_list_rcv, head, pt_prev, orig_dev);
5429 	else
5430 		list_for_each_entry_safe(skb, next, head, list) {
5431 			skb_list_del_init(skb);
5432 			pt_prev->func(skb, skb->dev, pt_prev, orig_dev);
5433 		}
5434 }
5435 
5436 static void __netif_receive_skb_list_core(struct list_head *head, bool pfmemalloc)
5437 {
5438 	/* Fast-path assumptions:
5439 	 * - There is no RX handler.
5440 	 * - Only one packet_type matches.
5441 	 * If either of these fails, we will end up doing some per-packet
5442 	 * processing in-line, then handling the 'last ptype' for the whole
5443 	 * sublist.  This can't cause out-of-order delivery to any single ptype,
5444 	 * because the 'last ptype' must be constant across the sublist, and all
5445 	 * other ptypes are handled per-packet.
5446 	 */
5447 	/* Current (common) ptype of sublist */
5448 	struct packet_type *pt_curr = NULL;
5449 	/* Current (common) orig_dev of sublist */
5450 	struct net_device *od_curr = NULL;
5451 	struct list_head sublist;
5452 	struct sk_buff *skb, *next;
5453 
5454 	INIT_LIST_HEAD(&sublist);
5455 	list_for_each_entry_safe(skb, next, head, list) {
5456 		struct net_device *orig_dev = skb->dev;
5457 		struct packet_type *pt_prev = NULL;
5458 
5459 		skb_list_del_init(skb);
5460 		__netif_receive_skb_core(&skb, pfmemalloc, &pt_prev);
5461 		if (!pt_prev)
5462 			continue;
5463 		if (pt_curr != pt_prev || od_curr != orig_dev) {
5464 			/* dispatch old sublist */
5465 			__netif_receive_skb_list_ptype(&sublist, pt_curr, od_curr);
5466 			/* start new sublist */
5467 			INIT_LIST_HEAD(&sublist);
5468 			pt_curr = pt_prev;
5469 			od_curr = orig_dev;
5470 		}
5471 		list_add_tail(&skb->list, &sublist);
5472 	}
5473 
5474 	/* dispatch final sublist */
5475 	__netif_receive_skb_list_ptype(&sublist, pt_curr, od_curr);
5476 }
5477 
5478 static int __netif_receive_skb(struct sk_buff *skb)
5479 {
5480 	int ret;
5481 
5482 	if (sk_memalloc_socks() && skb_pfmemalloc(skb)) {
5483 		unsigned int noreclaim_flag;
5484 
5485 		/*
5486 		 * PFMEMALLOC skbs are special, they should
5487 		 * - be delivered to SOCK_MEMALLOC sockets only
5488 		 * - stay away from userspace
5489 		 * - have bounded memory usage
5490 		 *
5491 		 * Use PF_MEMALLOC as this saves us from propagating the allocation
5492 		 * context down to all allocation sites.
5493 		 */
5494 		noreclaim_flag = memalloc_noreclaim_save();
5495 		ret = __netif_receive_skb_one_core(skb, true);
5496 		memalloc_noreclaim_restore(noreclaim_flag);
5497 	} else
5498 		ret = __netif_receive_skb_one_core(skb, false);
5499 
5500 	return ret;
5501 }
5502 
5503 static void __netif_receive_skb_list(struct list_head *head)
5504 {
5505 	unsigned long noreclaim_flag = 0;
5506 	struct sk_buff *skb, *next;
5507 	bool pfmemalloc = false; /* Is current sublist PF_MEMALLOC? */
5508 
5509 	list_for_each_entry_safe(skb, next, head, list) {
5510 		if ((sk_memalloc_socks() && skb_pfmemalloc(skb)) != pfmemalloc) {
5511 			struct list_head sublist;
5512 
5513 			/* Handle the previous sublist */
5514 			list_cut_before(&sublist, head, &skb->list);
5515 			if (!list_empty(&sublist))
5516 				__netif_receive_skb_list_core(&sublist, pfmemalloc);
5517 			pfmemalloc = !pfmemalloc;
5518 			/* See comments in __netif_receive_skb */
5519 			if (pfmemalloc)
5520 				noreclaim_flag = memalloc_noreclaim_save();
5521 			else
5522 				memalloc_noreclaim_restore(noreclaim_flag);
5523 		}
5524 	}
5525 	/* Handle the remaining sublist */
5526 	if (!list_empty(head))
5527 		__netif_receive_skb_list_core(head, pfmemalloc);
5528 	/* Restore pflags */
5529 	if (pfmemalloc)
5530 		memalloc_noreclaim_restore(noreclaim_flag);
5531 }
5532 
5533 static int generic_xdp_install(struct net_device *dev, struct netdev_bpf *xdp)
5534 {
5535 	struct bpf_prog *old = rtnl_dereference(dev->xdp_prog);
5536 	struct bpf_prog *new = xdp->prog;
5537 	int ret = 0;
5538 
5539 	if (new) {
5540 		u32 i;
5541 
5542 		mutex_lock(&new->aux->used_maps_mutex);
5543 
5544 		/* generic XDP does not work with DEVMAPs that can
5545 		 * have a bpf_prog installed on an entry
5546 		 */
5547 		for (i = 0; i < new->aux->used_map_cnt; i++) {
5548 			if (dev_map_can_have_prog(new->aux->used_maps[i]) ||
5549 			    cpu_map_prog_allowed(new->aux->used_maps[i])) {
5550 				mutex_unlock(&new->aux->used_maps_mutex);
5551 				return -EINVAL;
5552 			}
5553 		}
5554 
5555 		mutex_unlock(&new->aux->used_maps_mutex);
5556 	}
5557 
5558 	switch (xdp->command) {
5559 	case XDP_SETUP_PROG:
5560 		rcu_assign_pointer(dev->xdp_prog, new);
5561 		if (old)
5562 			bpf_prog_put(old);
5563 
5564 		if (old && !new) {
5565 			static_branch_dec(&generic_xdp_needed_key);
5566 		} else if (new && !old) {
5567 			static_branch_inc(&generic_xdp_needed_key);
5568 			dev_disable_lro(dev);
5569 			dev_disable_gro_hw(dev);
5570 		}
5571 		break;
5572 
5573 	default:
5574 		ret = -EINVAL;
5575 		break;
5576 	}
5577 
5578 	return ret;
5579 }
5580 
5581 static int netif_receive_skb_internal(struct sk_buff *skb)
5582 {
5583 	int ret;
5584 
5585 	net_timestamp_check(netdev_tstamp_prequeue, skb);
5586 
5587 	if (skb_defer_rx_timestamp(skb))
5588 		return NET_RX_SUCCESS;
5589 
5590 	rcu_read_lock();
5591 #ifdef CONFIG_RPS
5592 	if (static_branch_unlikely(&rps_needed)) {
5593 		struct rps_dev_flow voidflow, *rflow = &voidflow;
5594 		int cpu = get_rps_cpu(skb->dev, skb, &rflow);
5595 
5596 		if (cpu >= 0) {
5597 			ret = enqueue_to_backlog(skb, cpu, &rflow->last_qtail);
5598 			rcu_read_unlock();
5599 			return ret;
5600 		}
5601 	}
5602 #endif
5603 	ret = __netif_receive_skb(skb);
5604 	rcu_read_unlock();
5605 	return ret;
5606 }
5607 
5608 static void netif_receive_skb_list_internal(struct list_head *head)
5609 {
5610 	struct sk_buff *skb, *next;
5611 	struct list_head sublist;
5612 
5613 	INIT_LIST_HEAD(&sublist);
5614 	list_for_each_entry_safe(skb, next, head, list) {
5615 		net_timestamp_check(netdev_tstamp_prequeue, skb);
5616 		skb_list_del_init(skb);
5617 		if (!skb_defer_rx_timestamp(skb))
5618 			list_add_tail(&skb->list, &sublist);
5619 	}
5620 	list_splice_init(&sublist, head);
5621 
5622 	rcu_read_lock();
5623 #ifdef CONFIG_RPS
5624 	if (static_branch_unlikely(&rps_needed)) {
5625 		list_for_each_entry_safe(skb, next, head, list) {
5626 			struct rps_dev_flow voidflow, *rflow = &voidflow;
5627 			int cpu = get_rps_cpu(skb->dev, skb, &rflow);
5628 
5629 			if (cpu >= 0) {
5630 				/* Will be handled, remove from list */
5631 				skb_list_del_init(skb);
5632 				enqueue_to_backlog(skb, cpu, &rflow->last_qtail);
5633 			}
5634 		}
5635 	}
5636 #endif
5637 	__netif_receive_skb_list(head);
5638 	rcu_read_unlock();
5639 }
5640 
5641 /**
5642  *	netif_receive_skb - process receive buffer from network
5643  *	@skb: buffer to process
5644  *
5645  *	netif_receive_skb() is the main receive data processing function.
5646  *	It always succeeds. The buffer may be dropped during processing
5647  *	for congestion control or by the protocol layers.
5648  *
5649  *	This function may only be called from softirq context and interrupts
5650  *	should be enabled.
5651  *
5652  *	Return values (usually ignored):
5653  *	NET_RX_SUCCESS: no congestion
5654  *	NET_RX_DROP: packet was dropped
5655  */
5656 int netif_receive_skb(struct sk_buff *skb)
5657 {
5658 	int ret;
5659 
5660 	trace_netif_receive_skb_entry(skb);
5661 
5662 	ret = netif_receive_skb_internal(skb);
5663 	trace_netif_receive_skb_exit(ret);
5664 
5665 	return ret;
5666 }
5667 EXPORT_SYMBOL(netif_receive_skb);
5668 
5669 /**
5670  *	netif_receive_skb_list - process many receive buffers from network
5671  *	@head: list of skbs to process.
5672  *
5673  *	Since return value of netif_receive_skb() is normally ignored, and
5674  *	wouldn't be meaningful for a list, this function returns void.
5675  *
5676  *	This function may only be called from softirq context and interrupts
5677  *	should be enabled.
5678  */
5679 void netif_receive_skb_list(struct list_head *head)
5680 {
5681 	struct sk_buff *skb;
5682 
5683 	if (list_empty(head))
5684 		return;
5685 	if (trace_netif_receive_skb_list_entry_enabled()) {
5686 		list_for_each_entry(skb, head, list)
5687 			trace_netif_receive_skb_list_entry(skb);
5688 	}
5689 	netif_receive_skb_list_internal(head);
5690 	trace_netif_receive_skb_list_exit(0);
5691 }
5692 EXPORT_SYMBOL(netif_receive_skb_list);
5693 
5694 static DEFINE_PER_CPU(struct work_struct, flush_works);
5695 
5696 /* Network device is going away, flush any packets still pending */
5697 static void flush_backlog(struct work_struct *work)
5698 {
5699 	struct sk_buff *skb, *tmp;
5700 	struct softnet_data *sd;
5701 
5702 	local_bh_disable();
5703 	sd = this_cpu_ptr(&softnet_data);
5704 
5705 	local_irq_disable();
5706 	rps_lock(sd);
5707 	skb_queue_walk_safe(&sd->input_pkt_queue, skb, tmp) {
5708 		if (skb->dev->reg_state == NETREG_UNREGISTERING) {
5709 			__skb_unlink(skb, &sd->input_pkt_queue);
5710 			dev_kfree_skb_irq(skb);
5711 			input_queue_head_incr(sd);
5712 		}
5713 	}
5714 	rps_unlock(sd);
5715 	local_irq_enable();
5716 
5717 	skb_queue_walk_safe(&sd->process_queue, skb, tmp) {
5718 		if (skb->dev->reg_state == NETREG_UNREGISTERING) {
5719 			__skb_unlink(skb, &sd->process_queue);
5720 			kfree_skb(skb);
5721 			input_queue_head_incr(sd);
5722 		}
5723 	}
5724 	local_bh_enable();
5725 }
5726 
5727 static bool flush_required(int cpu)
5728 {
5729 #if IS_ENABLED(CONFIG_RPS)
5730 	struct softnet_data *sd = &per_cpu(softnet_data, cpu);
5731 	bool do_flush;
5732 
5733 	local_irq_disable();
5734 	rps_lock(sd);
5735 
5736 	/* as insertion into process_queue happens with the rps lock held,
5737 	 * process_queue access may race only with dequeue
5738 	 */
5739 	do_flush = !skb_queue_empty(&sd->input_pkt_queue) ||
5740 		   !skb_queue_empty_lockless(&sd->process_queue);
5741 	rps_unlock(sd);
5742 	local_irq_enable();
5743 
5744 	return do_flush;
5745 #endif
5746 	/* without RPS we can't safely check input_pkt_queue: during a
5747 	 * concurrent remote skb_queue_splice() we can detect as empty both
5748 	 * input_pkt_queue and process_queue even if the latter could end-up
5749 	 * containing a lot of packets.
5750 	 */
5751 	return true;
5752 }
5753 
5754 static void flush_all_backlogs(void)
5755 {
5756 	static cpumask_t flush_cpus;
5757 	unsigned int cpu;
5758 
5759 	/* since we are under rtnl lock protection we can use static data
5760 	 * for the cpumask and avoid allocating on stack the possibly
5761 	 * large mask
5762 	 */
5763 	ASSERT_RTNL();
5764 
5765 	get_online_cpus();
5766 
5767 	cpumask_clear(&flush_cpus);
5768 	for_each_online_cpu(cpu) {
5769 		if (flush_required(cpu)) {
5770 			queue_work_on(cpu, system_highpri_wq,
5771 				      per_cpu_ptr(&flush_works, cpu));
5772 			cpumask_set_cpu(cpu, &flush_cpus);
5773 		}
5774 	}
5775 
5776 	/* we can have in flight packet[s] on the cpus we are not flushing,
5777 	 * synchronize_net() in unregister_netdevice_many() will take care of
5778 	 * them
5779 	 */
5780 	for_each_cpu(cpu, &flush_cpus)
5781 		flush_work(per_cpu_ptr(&flush_works, cpu));
5782 
5783 	put_online_cpus();
5784 }
5785 
5786 /* Pass the currently batched GRO_NORMAL SKBs up to the stack. */
5787 static void gro_normal_list(struct napi_struct *napi)
5788 {
5789 	if (!napi->rx_count)
5790 		return;
5791 	netif_receive_skb_list_internal(&napi->rx_list);
5792 	INIT_LIST_HEAD(&napi->rx_list);
5793 	napi->rx_count = 0;
5794 }
5795 
5796 /* Queue one GRO_NORMAL SKB up for list processing. If batch size exceeded,
5797  * pass the whole batch up to the stack.
5798  */
5799 static void gro_normal_one(struct napi_struct *napi, struct sk_buff *skb, int segs)
5800 {
5801 	list_add_tail(&skb->list, &napi->rx_list);
5802 	napi->rx_count += segs;
5803 	if (napi->rx_count >= gro_normal_batch)
5804 		gro_normal_list(napi);
5805 }
5806 
5807 static int napi_gro_complete(struct napi_struct *napi, struct sk_buff *skb)
5808 {
5809 	struct packet_offload *ptype;
5810 	__be16 type = skb->protocol;
5811 	struct list_head *head = &offload_base;
5812 	int err = -ENOENT;
5813 
5814 	BUILD_BUG_ON(sizeof(struct napi_gro_cb) > sizeof(skb->cb));
5815 
5816 	if (NAPI_GRO_CB(skb)->count == 1) {
5817 		skb_shinfo(skb)->gso_size = 0;
5818 		goto out;
5819 	}
5820 
5821 	rcu_read_lock();
5822 	list_for_each_entry_rcu(ptype, head, list) {
5823 		if (ptype->type != type || !ptype->callbacks.gro_complete)
5824 			continue;
5825 
5826 		err = INDIRECT_CALL_INET(ptype->callbacks.gro_complete,
5827 					 ipv6_gro_complete, inet_gro_complete,
5828 					 skb, 0);
5829 		break;
5830 	}
5831 	rcu_read_unlock();
5832 
5833 	if (err) {
5834 		WARN_ON(&ptype->list == head);
5835 		kfree_skb(skb);
5836 		return NET_RX_SUCCESS;
5837 	}
5838 
5839 out:
5840 	gro_normal_one(napi, skb, NAPI_GRO_CB(skb)->count);
5841 	return NET_RX_SUCCESS;
5842 }
5843 
5844 static void __napi_gro_flush_chain(struct napi_struct *napi, u32 index,
5845 				   bool flush_old)
5846 {
5847 	struct list_head *head = &napi->gro_hash[index].list;
5848 	struct sk_buff *skb, *p;
5849 
5850 	list_for_each_entry_safe_reverse(skb, p, head, list) {
5851 		if (flush_old && NAPI_GRO_CB(skb)->age == jiffies)
5852 			return;
5853 		skb_list_del_init(skb);
5854 		napi_gro_complete(napi, skb);
5855 		napi->gro_hash[index].count--;
5856 	}
5857 
5858 	if (!napi->gro_hash[index].count)
5859 		__clear_bit(index, &napi->gro_bitmask);
5860 }
5861 
5862 /* napi->gro_hash[].list contains packets ordered by age.
5863  * youngest packets at the head of it.
5864  * Complete skbs in reverse order to reduce latencies.
5865  */
5866 void napi_gro_flush(struct napi_struct *napi, bool flush_old)
5867 {
5868 	unsigned long bitmask = napi->gro_bitmask;
5869 	unsigned int i, base = ~0U;
5870 
5871 	while ((i = ffs(bitmask)) != 0) {
5872 		bitmask >>= i;
5873 		base += i;
5874 		__napi_gro_flush_chain(napi, base, flush_old);
5875 	}
5876 }
5877 EXPORT_SYMBOL(napi_gro_flush);
5878 
5879 static struct list_head *gro_list_prepare(struct napi_struct *napi,
5880 					  struct sk_buff *skb)
5881 {
5882 	unsigned int maclen = skb->dev->hard_header_len;
5883 	u32 hash = skb_get_hash_raw(skb);
5884 	struct list_head *head;
5885 	struct sk_buff *p;
5886 
5887 	head = &napi->gro_hash[hash & (GRO_HASH_BUCKETS - 1)].list;
5888 	list_for_each_entry(p, head, list) {
5889 		unsigned long diffs;
5890 
5891 		NAPI_GRO_CB(p)->flush = 0;
5892 
5893 		if (hash != skb_get_hash_raw(p)) {
5894 			NAPI_GRO_CB(p)->same_flow = 0;
5895 			continue;
5896 		}
5897 
5898 		diffs = (unsigned long)p->dev ^ (unsigned long)skb->dev;
5899 		diffs |= skb_vlan_tag_present(p) ^ skb_vlan_tag_present(skb);
5900 		if (skb_vlan_tag_present(p))
5901 			diffs |= skb_vlan_tag_get(p) ^ skb_vlan_tag_get(skb);
5902 		diffs |= skb_metadata_dst_cmp(p, skb);
5903 		diffs |= skb_metadata_differs(p, skb);
5904 		if (maclen == ETH_HLEN)
5905 			diffs |= compare_ether_header(skb_mac_header(p),
5906 						      skb_mac_header(skb));
5907 		else if (!diffs)
5908 			diffs = memcmp(skb_mac_header(p),
5909 				       skb_mac_header(skb),
5910 				       maclen);
5911 		NAPI_GRO_CB(p)->same_flow = !diffs;
5912 	}
5913 
5914 	return head;
5915 }
5916 
5917 static void skb_gro_reset_offset(struct sk_buff *skb)
5918 {
5919 	const struct skb_shared_info *pinfo = skb_shinfo(skb);
5920 	const skb_frag_t *frag0 = &pinfo->frags[0];
5921 
5922 	NAPI_GRO_CB(skb)->data_offset = 0;
5923 	NAPI_GRO_CB(skb)->frag0 = NULL;
5924 	NAPI_GRO_CB(skb)->frag0_len = 0;
5925 
5926 	if (!skb_headlen(skb) && pinfo->nr_frags &&
5927 	    !PageHighMem(skb_frag_page(frag0))) {
5928 		NAPI_GRO_CB(skb)->frag0 = skb_frag_address(frag0);
5929 		NAPI_GRO_CB(skb)->frag0_len = min_t(unsigned int,
5930 						    skb_frag_size(frag0),
5931 						    skb->end - skb->tail);
5932 	}
5933 }
5934 
5935 static void gro_pull_from_frag0(struct sk_buff *skb, int grow)
5936 {
5937 	struct skb_shared_info *pinfo = skb_shinfo(skb);
5938 
5939 	BUG_ON(skb->end - skb->tail < grow);
5940 
5941 	memcpy(skb_tail_pointer(skb), NAPI_GRO_CB(skb)->frag0, grow);
5942 
5943 	skb->data_len -= grow;
5944 	skb->tail += grow;
5945 
5946 	skb_frag_off_add(&pinfo->frags[0], grow);
5947 	skb_frag_size_sub(&pinfo->frags[0], grow);
5948 
5949 	if (unlikely(!skb_frag_size(&pinfo->frags[0]))) {
5950 		skb_frag_unref(skb, 0);
5951 		memmove(pinfo->frags, pinfo->frags + 1,
5952 			--pinfo->nr_frags * sizeof(pinfo->frags[0]));
5953 	}
5954 }
5955 
5956 static void gro_flush_oldest(struct napi_struct *napi, struct list_head *head)
5957 {
5958 	struct sk_buff *oldest;
5959 
5960 	oldest = list_last_entry(head, struct sk_buff, list);
5961 
5962 	/* We are called with head length >= MAX_GRO_SKBS, so this is
5963 	 * impossible.
5964 	 */
5965 	if (WARN_ON_ONCE(!oldest))
5966 		return;
5967 
5968 	/* Do not adjust napi->gro_hash[].count, caller is adding a new
5969 	 * SKB to the chain.
5970 	 */
5971 	skb_list_del_init(oldest);
5972 	napi_gro_complete(napi, oldest);
5973 }
5974 
5975 static enum gro_result dev_gro_receive(struct napi_struct *napi, struct sk_buff *skb)
5976 {
5977 	u32 hash = skb_get_hash_raw(skb) & (GRO_HASH_BUCKETS - 1);
5978 	struct list_head *head = &offload_base;
5979 	struct packet_offload *ptype;
5980 	__be16 type = skb->protocol;
5981 	struct list_head *gro_head;
5982 	struct sk_buff *pp = NULL;
5983 	enum gro_result ret;
5984 	int same_flow;
5985 	int grow;
5986 
5987 	if (netif_elide_gro(skb->dev))
5988 		goto normal;
5989 
5990 	gro_head = gro_list_prepare(napi, skb);
5991 
5992 	rcu_read_lock();
5993 	list_for_each_entry_rcu(ptype, head, list) {
5994 		if (ptype->type != type || !ptype->callbacks.gro_receive)
5995 			continue;
5996 
5997 		skb_set_network_header(skb, skb_gro_offset(skb));
5998 		skb_reset_mac_len(skb);
5999 		NAPI_GRO_CB(skb)->same_flow = 0;
6000 		NAPI_GRO_CB(skb)->flush = skb_is_gso(skb) || skb_has_frag_list(skb);
6001 		NAPI_GRO_CB(skb)->free = 0;
6002 		NAPI_GRO_CB(skb)->encap_mark = 0;
6003 		NAPI_GRO_CB(skb)->recursion_counter = 0;
6004 		NAPI_GRO_CB(skb)->is_fou = 0;
6005 		NAPI_GRO_CB(skb)->is_atomic = 1;
6006 		NAPI_GRO_CB(skb)->gro_remcsum_start = 0;
6007 
6008 		/* Setup for GRO checksum validation */
6009 		switch (skb->ip_summed) {
6010 		case CHECKSUM_COMPLETE:
6011 			NAPI_GRO_CB(skb)->csum = skb->csum;
6012 			NAPI_GRO_CB(skb)->csum_valid = 1;
6013 			NAPI_GRO_CB(skb)->csum_cnt = 0;
6014 			break;
6015 		case CHECKSUM_UNNECESSARY:
6016 			NAPI_GRO_CB(skb)->csum_cnt = skb->csum_level + 1;
6017 			NAPI_GRO_CB(skb)->csum_valid = 0;
6018 			break;
6019 		default:
6020 			NAPI_GRO_CB(skb)->csum_cnt = 0;
6021 			NAPI_GRO_CB(skb)->csum_valid = 0;
6022 		}
6023 
6024 		pp = INDIRECT_CALL_INET(ptype->callbacks.gro_receive,
6025 					ipv6_gro_receive, inet_gro_receive,
6026 					gro_head, skb);
6027 		break;
6028 	}
6029 	rcu_read_unlock();
6030 
6031 	if (&ptype->list == head)
6032 		goto normal;
6033 
6034 	if (PTR_ERR(pp) == -EINPROGRESS) {
6035 		ret = GRO_CONSUMED;
6036 		goto ok;
6037 	}
6038 
6039 	same_flow = NAPI_GRO_CB(skb)->same_flow;
6040 	ret = NAPI_GRO_CB(skb)->free ? GRO_MERGED_FREE : GRO_MERGED;
6041 
6042 	if (pp) {
6043 		skb_list_del_init(pp);
6044 		napi_gro_complete(napi, pp);
6045 		napi->gro_hash[hash].count--;
6046 	}
6047 
6048 	if (same_flow)
6049 		goto ok;
6050 
6051 	if (NAPI_GRO_CB(skb)->flush)
6052 		goto normal;
6053 
6054 	if (unlikely(napi->gro_hash[hash].count >= MAX_GRO_SKBS)) {
6055 		gro_flush_oldest(napi, gro_head);
6056 	} else {
6057 		napi->gro_hash[hash].count++;
6058 	}
6059 	NAPI_GRO_CB(skb)->count = 1;
6060 	NAPI_GRO_CB(skb)->age = jiffies;
6061 	NAPI_GRO_CB(skb)->last = skb;
6062 	skb_shinfo(skb)->gso_size = skb_gro_len(skb);
6063 	list_add(&skb->list, gro_head);
6064 	ret = GRO_HELD;
6065 
6066 pull:
6067 	grow = skb_gro_offset(skb) - skb_headlen(skb);
6068 	if (grow > 0)
6069 		gro_pull_from_frag0(skb, grow);
6070 ok:
6071 	if (napi->gro_hash[hash].count) {
6072 		if (!test_bit(hash, &napi->gro_bitmask))
6073 			__set_bit(hash, &napi->gro_bitmask);
6074 	} else if (test_bit(hash, &napi->gro_bitmask)) {
6075 		__clear_bit(hash, &napi->gro_bitmask);
6076 	}
6077 
6078 	return ret;
6079 
6080 normal:
6081 	ret = GRO_NORMAL;
6082 	goto pull;
6083 }
6084 
6085 struct packet_offload *gro_find_receive_by_type(__be16 type)
6086 {
6087 	struct list_head *offload_head = &offload_base;
6088 	struct packet_offload *ptype;
6089 
6090 	list_for_each_entry_rcu(ptype, offload_head, list) {
6091 		if (ptype->type != type || !ptype->callbacks.gro_receive)
6092 			continue;
6093 		return ptype;
6094 	}
6095 	return NULL;
6096 }
6097 EXPORT_SYMBOL(gro_find_receive_by_type);
6098 
6099 struct packet_offload *gro_find_complete_by_type(__be16 type)
6100 {
6101 	struct list_head *offload_head = &offload_base;
6102 	struct packet_offload *ptype;
6103 
6104 	list_for_each_entry_rcu(ptype, offload_head, list) {
6105 		if (ptype->type != type || !ptype->callbacks.gro_complete)
6106 			continue;
6107 		return ptype;
6108 	}
6109 	return NULL;
6110 }
6111 EXPORT_SYMBOL(gro_find_complete_by_type);
6112 
6113 static gro_result_t napi_skb_finish(struct napi_struct *napi,
6114 				    struct sk_buff *skb,
6115 				    gro_result_t ret)
6116 {
6117 	switch (ret) {
6118 	case GRO_NORMAL:
6119 		gro_normal_one(napi, skb, 1);
6120 		break;
6121 
6122 	case GRO_MERGED_FREE:
6123 		if (NAPI_GRO_CB(skb)->free == NAPI_GRO_FREE_STOLEN_HEAD)
6124 			napi_skb_free_stolen_head(skb);
6125 		else
6126 			__kfree_skb_defer(skb);
6127 		break;
6128 
6129 	case GRO_HELD:
6130 	case GRO_MERGED:
6131 	case GRO_CONSUMED:
6132 		break;
6133 	}
6134 
6135 	return ret;
6136 }
6137 
6138 gro_result_t napi_gro_receive(struct napi_struct *napi, struct sk_buff *skb)
6139 {
6140 	gro_result_t ret;
6141 
6142 	skb_mark_napi_id(skb, napi);
6143 	trace_napi_gro_receive_entry(skb);
6144 
6145 	skb_gro_reset_offset(skb);
6146 
6147 	ret = napi_skb_finish(napi, skb, dev_gro_receive(napi, skb));
6148 	trace_napi_gro_receive_exit(ret);
6149 
6150 	return ret;
6151 }
6152 EXPORT_SYMBOL(napi_gro_receive);
6153 
6154 static void napi_reuse_skb(struct napi_struct *napi, struct sk_buff *skb)
6155 {
6156 	if (unlikely(skb->pfmemalloc)) {
6157 		consume_skb(skb);
6158 		return;
6159 	}
6160 	__skb_pull(skb, skb_headlen(skb));
6161 	/* restore the reserve we had after netdev_alloc_skb_ip_align() */
6162 	skb_reserve(skb, NET_SKB_PAD + NET_IP_ALIGN - skb_headroom(skb));
6163 	__vlan_hwaccel_clear_tag(skb);
6164 	skb->dev = napi->dev;
6165 	skb->skb_iif = 0;
6166 
6167 	/* eth_type_trans() assumes pkt_type is PACKET_HOST */
6168 	skb->pkt_type = PACKET_HOST;
6169 
6170 	skb->encapsulation = 0;
6171 	skb_shinfo(skb)->gso_type = 0;
6172 	skb->truesize = SKB_TRUESIZE(skb_end_offset(skb));
6173 	skb_ext_reset(skb);
6174 
6175 	napi->skb = skb;
6176 }
6177 
6178 struct sk_buff *napi_get_frags(struct napi_struct *napi)
6179 {
6180 	struct sk_buff *skb = napi->skb;
6181 
6182 	if (!skb) {
6183 		skb = napi_alloc_skb(napi, GRO_MAX_HEAD);
6184 		if (skb) {
6185 			napi->skb = skb;
6186 			skb_mark_napi_id(skb, napi);
6187 		}
6188 	}
6189 	return skb;
6190 }
6191 EXPORT_SYMBOL(napi_get_frags);
6192 
6193 static gro_result_t napi_frags_finish(struct napi_struct *napi,
6194 				      struct sk_buff *skb,
6195 				      gro_result_t ret)
6196 {
6197 	switch (ret) {
6198 	case GRO_NORMAL:
6199 	case GRO_HELD:
6200 		__skb_push(skb, ETH_HLEN);
6201 		skb->protocol = eth_type_trans(skb, skb->dev);
6202 		if (ret == GRO_NORMAL)
6203 			gro_normal_one(napi, skb, 1);
6204 		break;
6205 
6206 	case GRO_MERGED_FREE:
6207 		if (NAPI_GRO_CB(skb)->free == NAPI_GRO_FREE_STOLEN_HEAD)
6208 			napi_skb_free_stolen_head(skb);
6209 		else
6210 			napi_reuse_skb(napi, skb);
6211 		break;
6212 
6213 	case GRO_MERGED:
6214 	case GRO_CONSUMED:
6215 		break;
6216 	}
6217 
6218 	return ret;
6219 }
6220 
6221 /* Upper GRO stack assumes network header starts at gro_offset=0
6222  * Drivers could call both napi_gro_frags() and napi_gro_receive()
6223  * We copy ethernet header into skb->data to have a common layout.
6224  */
6225 static struct sk_buff *napi_frags_skb(struct napi_struct *napi)
6226 {
6227 	struct sk_buff *skb = napi->skb;
6228 	const struct ethhdr *eth;
6229 	unsigned int hlen = sizeof(*eth);
6230 
6231 	napi->skb = NULL;
6232 
6233 	skb_reset_mac_header(skb);
6234 	skb_gro_reset_offset(skb);
6235 
6236 	if (unlikely(skb_gro_header_hard(skb, hlen))) {
6237 		eth = skb_gro_header_slow(skb, hlen, 0);
6238 		if (unlikely(!eth)) {
6239 			net_warn_ratelimited("%s: dropping impossible skb from %s\n",
6240 					     __func__, napi->dev->name);
6241 			napi_reuse_skb(napi, skb);
6242 			return NULL;
6243 		}
6244 	} else {
6245 		eth = (const struct ethhdr *)skb->data;
6246 		gro_pull_from_frag0(skb, hlen);
6247 		NAPI_GRO_CB(skb)->frag0 += hlen;
6248 		NAPI_GRO_CB(skb)->frag0_len -= hlen;
6249 	}
6250 	__skb_pull(skb, hlen);
6251 
6252 	/*
6253 	 * This works because the only protocols we care about don't require
6254 	 * special handling.
6255 	 * We'll fix it up properly in napi_frags_finish()
6256 	 */
6257 	skb->protocol = eth->h_proto;
6258 
6259 	return skb;
6260 }
6261 
6262 gro_result_t napi_gro_frags(struct napi_struct *napi)
6263 {
6264 	gro_result_t ret;
6265 	struct sk_buff *skb = napi_frags_skb(napi);
6266 
6267 	trace_napi_gro_frags_entry(skb);
6268 
6269 	ret = napi_frags_finish(napi, skb, dev_gro_receive(napi, skb));
6270 	trace_napi_gro_frags_exit(ret);
6271 
6272 	return ret;
6273 }
6274 EXPORT_SYMBOL(napi_gro_frags);
6275 
6276 /* Compute the checksum from gro_offset and return the folded value
6277  * after adding in any pseudo checksum.
6278  */
6279 __sum16 __skb_gro_checksum_complete(struct sk_buff *skb)
6280 {
6281 	__wsum wsum;
6282 	__sum16 sum;
6283 
6284 	wsum = skb_checksum(skb, skb_gro_offset(skb), skb_gro_len(skb), 0);
6285 
6286 	/* NAPI_GRO_CB(skb)->csum holds pseudo checksum */
6287 	sum = csum_fold(csum_add(NAPI_GRO_CB(skb)->csum, wsum));
6288 	/* See comments in __skb_checksum_complete(). */
6289 	if (likely(!sum)) {
6290 		if (unlikely(skb->ip_summed == CHECKSUM_COMPLETE) &&
6291 		    !skb->csum_complete_sw)
6292 			netdev_rx_csum_fault(skb->dev, skb);
6293 	}
6294 
6295 	NAPI_GRO_CB(skb)->csum = wsum;
6296 	NAPI_GRO_CB(skb)->csum_valid = 1;
6297 
6298 	return sum;
6299 }
6300 EXPORT_SYMBOL(__skb_gro_checksum_complete);
6301 
6302 static void net_rps_send_ipi(struct softnet_data *remsd)
6303 {
6304 #ifdef CONFIG_RPS
6305 	while (remsd) {
6306 		struct softnet_data *next = remsd->rps_ipi_next;
6307 
6308 		if (cpu_online(remsd->cpu))
6309 			smp_call_function_single_async(remsd->cpu, &remsd->csd);
6310 		remsd = next;
6311 	}
6312 #endif
6313 }
6314 
6315 /*
6316  * net_rps_action_and_irq_enable sends any pending IPI's for rps.
6317  * Note: called with local irq disabled, but exits with local irq enabled.
6318  */
6319 static void net_rps_action_and_irq_enable(struct softnet_data *sd)
6320 {
6321 #ifdef CONFIG_RPS
6322 	struct softnet_data *remsd = sd->rps_ipi_list;
6323 
6324 	if (remsd) {
6325 		sd->rps_ipi_list = NULL;
6326 
6327 		local_irq_enable();
6328 
6329 		/* Send pending IPI's to kick RPS processing on remote cpus. */
6330 		net_rps_send_ipi(remsd);
6331 	} else
6332 #endif
6333 		local_irq_enable();
6334 }
6335 
6336 static bool sd_has_rps_ipi_waiting(struct softnet_data *sd)
6337 {
6338 #ifdef CONFIG_RPS
6339 	return sd->rps_ipi_list != NULL;
6340 #else
6341 	return false;
6342 #endif
6343 }
6344 
6345 static int process_backlog(struct napi_struct *napi, int quota)
6346 {
6347 	struct softnet_data *sd = container_of(napi, struct softnet_data, backlog);
6348 	bool again = true;
6349 	int work = 0;
6350 
6351 	/* Check if we have pending ipi, its better to send them now,
6352 	 * not waiting net_rx_action() end.
6353 	 */
6354 	if (sd_has_rps_ipi_waiting(sd)) {
6355 		local_irq_disable();
6356 		net_rps_action_and_irq_enable(sd);
6357 	}
6358 
6359 	napi->weight = dev_rx_weight;
6360 	while (again) {
6361 		struct sk_buff *skb;
6362 
6363 		while ((skb = __skb_dequeue(&sd->process_queue))) {
6364 			rcu_read_lock();
6365 			__netif_receive_skb(skb);
6366 			rcu_read_unlock();
6367 			input_queue_head_incr(sd);
6368 			if (++work >= quota)
6369 				return work;
6370 
6371 		}
6372 
6373 		local_irq_disable();
6374 		rps_lock(sd);
6375 		if (skb_queue_empty(&sd->input_pkt_queue)) {
6376 			/*
6377 			 * Inline a custom version of __napi_complete().
6378 			 * only current cpu owns and manipulates this napi,
6379 			 * and NAPI_STATE_SCHED is the only possible flag set
6380 			 * on backlog.
6381 			 * We can use a plain write instead of clear_bit(),
6382 			 * and we dont need an smp_mb() memory barrier.
6383 			 */
6384 			napi->state = 0;
6385 			again = false;
6386 		} else {
6387 			skb_queue_splice_tail_init(&sd->input_pkt_queue,
6388 						   &sd->process_queue);
6389 		}
6390 		rps_unlock(sd);
6391 		local_irq_enable();
6392 	}
6393 
6394 	return work;
6395 }
6396 
6397 /**
6398  * __napi_schedule - schedule for receive
6399  * @n: entry to schedule
6400  *
6401  * The entry's receive function will be scheduled to run.
6402  * Consider using __napi_schedule_irqoff() if hard irqs are masked.
6403  */
6404 void __napi_schedule(struct napi_struct *n)
6405 {
6406 	unsigned long flags;
6407 
6408 	local_irq_save(flags);
6409 	____napi_schedule(this_cpu_ptr(&softnet_data), n);
6410 	local_irq_restore(flags);
6411 }
6412 EXPORT_SYMBOL(__napi_schedule);
6413 
6414 /**
6415  *	napi_schedule_prep - check if napi can be scheduled
6416  *	@n: napi context
6417  *
6418  * Test if NAPI routine is already running, and if not mark
6419  * it as running.  This is used as a condition variable to
6420  * insure only one NAPI poll instance runs.  We also make
6421  * sure there is no pending NAPI disable.
6422  */
6423 bool napi_schedule_prep(struct napi_struct *n)
6424 {
6425 	unsigned long val, new;
6426 
6427 	do {
6428 		val = READ_ONCE(n->state);
6429 		if (unlikely(val & NAPIF_STATE_DISABLE))
6430 			return false;
6431 		new = val | NAPIF_STATE_SCHED;
6432 
6433 		/* Sets STATE_MISSED bit if STATE_SCHED was already set
6434 		 * This was suggested by Alexander Duyck, as compiler
6435 		 * emits better code than :
6436 		 * if (val & NAPIF_STATE_SCHED)
6437 		 *     new |= NAPIF_STATE_MISSED;
6438 		 */
6439 		new |= (val & NAPIF_STATE_SCHED) / NAPIF_STATE_SCHED *
6440 						   NAPIF_STATE_MISSED;
6441 	} while (cmpxchg(&n->state, val, new) != val);
6442 
6443 	return !(val & NAPIF_STATE_SCHED);
6444 }
6445 EXPORT_SYMBOL(napi_schedule_prep);
6446 
6447 /**
6448  * __napi_schedule_irqoff - schedule for receive
6449  * @n: entry to schedule
6450  *
6451  * Variant of __napi_schedule() assuming hard irqs are masked
6452  */
6453 void __napi_schedule_irqoff(struct napi_struct *n)
6454 {
6455 	____napi_schedule(this_cpu_ptr(&softnet_data), n);
6456 }
6457 EXPORT_SYMBOL(__napi_schedule_irqoff);
6458 
6459 bool napi_complete_done(struct napi_struct *n, int work_done)
6460 {
6461 	unsigned long flags, val, new, timeout = 0;
6462 	bool ret = true;
6463 
6464 	/*
6465 	 * 1) Don't let napi dequeue from the cpu poll list
6466 	 *    just in case its running on a different cpu.
6467 	 * 2) If we are busy polling, do nothing here, we have
6468 	 *    the guarantee we will be called later.
6469 	 */
6470 	if (unlikely(n->state & (NAPIF_STATE_NPSVC |
6471 				 NAPIF_STATE_IN_BUSY_POLL)))
6472 		return false;
6473 
6474 	if (work_done) {
6475 		if (n->gro_bitmask)
6476 			timeout = READ_ONCE(n->dev->gro_flush_timeout);
6477 		n->defer_hard_irqs_count = READ_ONCE(n->dev->napi_defer_hard_irqs);
6478 	}
6479 	if (n->defer_hard_irqs_count > 0) {
6480 		n->defer_hard_irqs_count--;
6481 		timeout = READ_ONCE(n->dev->gro_flush_timeout);
6482 		if (timeout)
6483 			ret = false;
6484 	}
6485 	if (n->gro_bitmask) {
6486 		/* When the NAPI instance uses a timeout and keeps postponing
6487 		 * it, we need to bound somehow the time packets are kept in
6488 		 * the GRO layer
6489 		 */
6490 		napi_gro_flush(n, !!timeout);
6491 	}
6492 
6493 	gro_normal_list(n);
6494 
6495 	if (unlikely(!list_empty(&n->poll_list))) {
6496 		/* If n->poll_list is not empty, we need to mask irqs */
6497 		local_irq_save(flags);
6498 		list_del_init(&n->poll_list);
6499 		local_irq_restore(flags);
6500 	}
6501 
6502 	do {
6503 		val = READ_ONCE(n->state);
6504 
6505 		WARN_ON_ONCE(!(val & NAPIF_STATE_SCHED));
6506 
6507 		new = val & ~(NAPIF_STATE_MISSED | NAPIF_STATE_SCHED |
6508 			      NAPIF_STATE_SCHED_THREADED |
6509 			      NAPIF_STATE_PREFER_BUSY_POLL);
6510 
6511 		/* If STATE_MISSED was set, leave STATE_SCHED set,
6512 		 * because we will call napi->poll() one more time.
6513 		 * This C code was suggested by Alexander Duyck to help gcc.
6514 		 */
6515 		new |= (val & NAPIF_STATE_MISSED) / NAPIF_STATE_MISSED *
6516 						    NAPIF_STATE_SCHED;
6517 	} while (cmpxchg(&n->state, val, new) != val);
6518 
6519 	if (unlikely(val & NAPIF_STATE_MISSED)) {
6520 		__napi_schedule(n);
6521 		return false;
6522 	}
6523 
6524 	if (timeout)
6525 		hrtimer_start(&n->timer, ns_to_ktime(timeout),
6526 			      HRTIMER_MODE_REL_PINNED);
6527 	return ret;
6528 }
6529 EXPORT_SYMBOL(napi_complete_done);
6530 
6531 /* must be called under rcu_read_lock(), as we dont take a reference */
6532 static struct napi_struct *napi_by_id(unsigned int napi_id)
6533 {
6534 	unsigned int hash = napi_id % HASH_SIZE(napi_hash);
6535 	struct napi_struct *napi;
6536 
6537 	hlist_for_each_entry_rcu(napi, &napi_hash[hash], napi_hash_node)
6538 		if (napi->napi_id == napi_id)
6539 			return napi;
6540 
6541 	return NULL;
6542 }
6543 
6544 #if defined(CONFIG_NET_RX_BUSY_POLL)
6545 
6546 static void __busy_poll_stop(struct napi_struct *napi, bool skip_schedule)
6547 {
6548 	if (!skip_schedule) {
6549 		gro_normal_list(napi);
6550 		__napi_schedule(napi);
6551 		return;
6552 	}
6553 
6554 	if (napi->gro_bitmask) {
6555 		/* flush too old packets
6556 		 * If HZ < 1000, flush all packets.
6557 		 */
6558 		napi_gro_flush(napi, HZ >= 1000);
6559 	}
6560 
6561 	gro_normal_list(napi);
6562 	clear_bit(NAPI_STATE_SCHED, &napi->state);
6563 }
6564 
6565 static void busy_poll_stop(struct napi_struct *napi, void *have_poll_lock, bool prefer_busy_poll,
6566 			   u16 budget)
6567 {
6568 	bool skip_schedule = false;
6569 	unsigned long timeout;
6570 	int rc;
6571 
6572 	/* Busy polling means there is a high chance device driver hard irq
6573 	 * could not grab NAPI_STATE_SCHED, and that NAPI_STATE_MISSED was
6574 	 * set in napi_schedule_prep().
6575 	 * Since we are about to call napi->poll() once more, we can safely
6576 	 * clear NAPI_STATE_MISSED.
6577 	 *
6578 	 * Note: x86 could use a single "lock and ..." instruction
6579 	 * to perform these two clear_bit()
6580 	 */
6581 	clear_bit(NAPI_STATE_MISSED, &napi->state);
6582 	clear_bit(NAPI_STATE_IN_BUSY_POLL, &napi->state);
6583 
6584 	local_bh_disable();
6585 
6586 	if (prefer_busy_poll) {
6587 		napi->defer_hard_irqs_count = READ_ONCE(napi->dev->napi_defer_hard_irqs);
6588 		timeout = READ_ONCE(napi->dev->gro_flush_timeout);
6589 		if (napi->defer_hard_irqs_count && timeout) {
6590 			hrtimer_start(&napi->timer, ns_to_ktime(timeout), HRTIMER_MODE_REL_PINNED);
6591 			skip_schedule = true;
6592 		}
6593 	}
6594 
6595 	/* All we really want here is to re-enable device interrupts.
6596 	 * Ideally, a new ndo_busy_poll_stop() could avoid another round.
6597 	 */
6598 	rc = napi->poll(napi, budget);
6599 	/* We can't gro_normal_list() here, because napi->poll() might have
6600 	 * rearmed the napi (napi_complete_done()) in which case it could
6601 	 * already be running on another CPU.
6602 	 */
6603 	trace_napi_poll(napi, rc, budget);
6604 	netpoll_poll_unlock(have_poll_lock);
6605 	if (rc == budget)
6606 		__busy_poll_stop(napi, skip_schedule);
6607 	local_bh_enable();
6608 }
6609 
6610 void napi_busy_loop(unsigned int napi_id,
6611 		    bool (*loop_end)(void *, unsigned long),
6612 		    void *loop_end_arg, bool prefer_busy_poll, u16 budget)
6613 {
6614 	unsigned long start_time = loop_end ? busy_loop_current_time() : 0;
6615 	int (*napi_poll)(struct napi_struct *napi, int budget);
6616 	void *have_poll_lock = NULL;
6617 	struct napi_struct *napi;
6618 
6619 restart:
6620 	napi_poll = NULL;
6621 
6622 	rcu_read_lock();
6623 
6624 	napi = napi_by_id(napi_id);
6625 	if (!napi)
6626 		goto out;
6627 
6628 	preempt_disable();
6629 	for (;;) {
6630 		int work = 0;
6631 
6632 		local_bh_disable();
6633 		if (!napi_poll) {
6634 			unsigned long val = READ_ONCE(napi->state);
6635 
6636 			/* If multiple threads are competing for this napi,
6637 			 * we avoid dirtying napi->state as much as we can.
6638 			 */
6639 			if (val & (NAPIF_STATE_DISABLE | NAPIF_STATE_SCHED |
6640 				   NAPIF_STATE_IN_BUSY_POLL)) {
6641 				if (prefer_busy_poll)
6642 					set_bit(NAPI_STATE_PREFER_BUSY_POLL, &napi->state);
6643 				goto count;
6644 			}
6645 			if (cmpxchg(&napi->state, val,
6646 				    val | NAPIF_STATE_IN_BUSY_POLL |
6647 					  NAPIF_STATE_SCHED) != val) {
6648 				if (prefer_busy_poll)
6649 					set_bit(NAPI_STATE_PREFER_BUSY_POLL, &napi->state);
6650 				goto count;
6651 			}
6652 			have_poll_lock = netpoll_poll_lock(napi);
6653 			napi_poll = napi->poll;
6654 		}
6655 		work = napi_poll(napi, budget);
6656 		trace_napi_poll(napi, work, budget);
6657 		gro_normal_list(napi);
6658 count:
6659 		if (work > 0)
6660 			__NET_ADD_STATS(dev_net(napi->dev),
6661 					LINUX_MIB_BUSYPOLLRXPACKETS, work);
6662 		local_bh_enable();
6663 
6664 		if (!loop_end || loop_end(loop_end_arg, start_time))
6665 			break;
6666 
6667 		if (unlikely(need_resched())) {
6668 			if (napi_poll)
6669 				busy_poll_stop(napi, have_poll_lock, prefer_busy_poll, budget);
6670 			preempt_enable();
6671 			rcu_read_unlock();
6672 			cond_resched();
6673 			if (loop_end(loop_end_arg, start_time))
6674 				return;
6675 			goto restart;
6676 		}
6677 		cpu_relax();
6678 	}
6679 	if (napi_poll)
6680 		busy_poll_stop(napi, have_poll_lock, prefer_busy_poll, budget);
6681 	preempt_enable();
6682 out:
6683 	rcu_read_unlock();
6684 }
6685 EXPORT_SYMBOL(napi_busy_loop);
6686 
6687 #endif /* CONFIG_NET_RX_BUSY_POLL */
6688 
6689 static void napi_hash_add(struct napi_struct *napi)
6690 {
6691 	if (test_bit(NAPI_STATE_NO_BUSY_POLL, &napi->state))
6692 		return;
6693 
6694 	spin_lock(&napi_hash_lock);
6695 
6696 	/* 0..NR_CPUS range is reserved for sender_cpu use */
6697 	do {
6698 		if (unlikely(++napi_gen_id < MIN_NAPI_ID))
6699 			napi_gen_id = MIN_NAPI_ID;
6700 	} while (napi_by_id(napi_gen_id));
6701 	napi->napi_id = napi_gen_id;
6702 
6703 	hlist_add_head_rcu(&napi->napi_hash_node,
6704 			   &napi_hash[napi->napi_id % HASH_SIZE(napi_hash)]);
6705 
6706 	spin_unlock(&napi_hash_lock);
6707 }
6708 
6709 /* Warning : caller is responsible to make sure rcu grace period
6710  * is respected before freeing memory containing @napi
6711  */
6712 static void napi_hash_del(struct napi_struct *napi)
6713 {
6714 	spin_lock(&napi_hash_lock);
6715 
6716 	hlist_del_init_rcu(&napi->napi_hash_node);
6717 
6718 	spin_unlock(&napi_hash_lock);
6719 }
6720 
6721 static enum hrtimer_restart napi_watchdog(struct hrtimer *timer)
6722 {
6723 	struct napi_struct *napi;
6724 
6725 	napi = container_of(timer, struct napi_struct, timer);
6726 
6727 	/* Note : we use a relaxed variant of napi_schedule_prep() not setting
6728 	 * NAPI_STATE_MISSED, since we do not react to a device IRQ.
6729 	 */
6730 	if (!napi_disable_pending(napi) &&
6731 	    !test_and_set_bit(NAPI_STATE_SCHED, &napi->state)) {
6732 		clear_bit(NAPI_STATE_PREFER_BUSY_POLL, &napi->state);
6733 		__napi_schedule_irqoff(napi);
6734 	}
6735 
6736 	return HRTIMER_NORESTART;
6737 }
6738 
6739 static void init_gro_hash(struct napi_struct *napi)
6740 {
6741 	int i;
6742 
6743 	for (i = 0; i < GRO_HASH_BUCKETS; i++) {
6744 		INIT_LIST_HEAD(&napi->gro_hash[i].list);
6745 		napi->gro_hash[i].count = 0;
6746 	}
6747 	napi->gro_bitmask = 0;
6748 }
6749 
6750 int dev_set_threaded(struct net_device *dev, bool threaded)
6751 {
6752 	struct napi_struct *napi;
6753 	int err = 0;
6754 
6755 	if (dev->threaded == threaded)
6756 		return 0;
6757 
6758 	if (threaded) {
6759 		list_for_each_entry(napi, &dev->napi_list, dev_list) {
6760 			if (!napi->thread) {
6761 				err = napi_kthread_create(napi);
6762 				if (err) {
6763 					threaded = false;
6764 					break;
6765 				}
6766 			}
6767 		}
6768 	}
6769 
6770 	dev->threaded = threaded;
6771 
6772 	/* Make sure kthread is created before THREADED bit
6773 	 * is set.
6774 	 */
6775 	smp_mb__before_atomic();
6776 
6777 	/* Setting/unsetting threaded mode on a napi might not immediately
6778 	 * take effect, if the current napi instance is actively being
6779 	 * polled. In this case, the switch between threaded mode and
6780 	 * softirq mode will happen in the next round of napi_schedule().
6781 	 * This should not cause hiccups/stalls to the live traffic.
6782 	 */
6783 	list_for_each_entry(napi, &dev->napi_list, dev_list) {
6784 		if (threaded)
6785 			set_bit(NAPI_STATE_THREADED, &napi->state);
6786 		else
6787 			clear_bit(NAPI_STATE_THREADED, &napi->state);
6788 	}
6789 
6790 	return err;
6791 }
6792 
6793 void netif_napi_add(struct net_device *dev, struct napi_struct *napi,
6794 		    int (*poll)(struct napi_struct *, int), int weight)
6795 {
6796 	if (WARN_ON(test_and_set_bit(NAPI_STATE_LISTED, &napi->state)))
6797 		return;
6798 
6799 	INIT_LIST_HEAD(&napi->poll_list);
6800 	INIT_HLIST_NODE(&napi->napi_hash_node);
6801 	hrtimer_init(&napi->timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL_PINNED);
6802 	napi->timer.function = napi_watchdog;
6803 	init_gro_hash(napi);
6804 	napi->skb = NULL;
6805 	INIT_LIST_HEAD(&napi->rx_list);
6806 	napi->rx_count = 0;
6807 	napi->poll = poll;
6808 	if (weight > NAPI_POLL_WEIGHT)
6809 		netdev_err_once(dev, "%s() called with weight %d\n", __func__,
6810 				weight);
6811 	napi->weight = weight;
6812 	napi->dev = dev;
6813 #ifdef CONFIG_NETPOLL
6814 	napi->poll_owner = -1;
6815 #endif
6816 	set_bit(NAPI_STATE_SCHED, &napi->state);
6817 	set_bit(NAPI_STATE_NPSVC, &napi->state);
6818 	list_add_rcu(&napi->dev_list, &dev->napi_list);
6819 	napi_hash_add(napi);
6820 	/* Create kthread for this napi if dev->threaded is set.
6821 	 * Clear dev->threaded if kthread creation failed so that
6822 	 * threaded mode will not be enabled in napi_enable().
6823 	 */
6824 	if (dev->threaded && napi_kthread_create(napi))
6825 		dev->threaded = 0;
6826 }
6827 EXPORT_SYMBOL(netif_napi_add);
6828 
6829 void napi_disable(struct napi_struct *n)
6830 {
6831 	might_sleep();
6832 	set_bit(NAPI_STATE_DISABLE, &n->state);
6833 
6834 	while (test_and_set_bit(NAPI_STATE_SCHED, &n->state))
6835 		msleep(1);
6836 	while (test_and_set_bit(NAPI_STATE_NPSVC, &n->state))
6837 		msleep(1);
6838 
6839 	hrtimer_cancel(&n->timer);
6840 
6841 	clear_bit(NAPI_STATE_PREFER_BUSY_POLL, &n->state);
6842 	clear_bit(NAPI_STATE_DISABLE, &n->state);
6843 	clear_bit(NAPI_STATE_THREADED, &n->state);
6844 }
6845 EXPORT_SYMBOL(napi_disable);
6846 
6847 /**
6848  *	napi_enable - enable NAPI scheduling
6849  *	@n: NAPI context
6850  *
6851  * Resume NAPI from being scheduled on this context.
6852  * Must be paired with napi_disable.
6853  */
6854 void napi_enable(struct napi_struct *n)
6855 {
6856 	BUG_ON(!test_bit(NAPI_STATE_SCHED, &n->state));
6857 	smp_mb__before_atomic();
6858 	clear_bit(NAPI_STATE_SCHED, &n->state);
6859 	clear_bit(NAPI_STATE_NPSVC, &n->state);
6860 	if (n->dev->threaded && n->thread)
6861 		set_bit(NAPI_STATE_THREADED, &n->state);
6862 }
6863 EXPORT_SYMBOL(napi_enable);
6864 
6865 static void flush_gro_hash(struct napi_struct *napi)
6866 {
6867 	int i;
6868 
6869 	for (i = 0; i < GRO_HASH_BUCKETS; i++) {
6870 		struct sk_buff *skb, *n;
6871 
6872 		list_for_each_entry_safe(skb, n, &napi->gro_hash[i].list, list)
6873 			kfree_skb(skb);
6874 		napi->gro_hash[i].count = 0;
6875 	}
6876 }
6877 
6878 /* Must be called in process context */
6879 void __netif_napi_del(struct napi_struct *napi)
6880 {
6881 	if (!test_and_clear_bit(NAPI_STATE_LISTED, &napi->state))
6882 		return;
6883 
6884 	napi_hash_del(napi);
6885 	list_del_rcu(&napi->dev_list);
6886 	napi_free_frags(napi);
6887 
6888 	flush_gro_hash(napi);
6889 	napi->gro_bitmask = 0;
6890 
6891 	if (napi->thread) {
6892 		kthread_stop(napi->thread);
6893 		napi->thread = NULL;
6894 	}
6895 }
6896 EXPORT_SYMBOL(__netif_napi_del);
6897 
6898 static int __napi_poll(struct napi_struct *n, bool *repoll)
6899 {
6900 	int work, weight;
6901 
6902 	weight = n->weight;
6903 
6904 	/* This NAPI_STATE_SCHED test is for avoiding a race
6905 	 * with netpoll's poll_napi().  Only the entity which
6906 	 * obtains the lock and sees NAPI_STATE_SCHED set will
6907 	 * actually make the ->poll() call.  Therefore we avoid
6908 	 * accidentally calling ->poll() when NAPI is not scheduled.
6909 	 */
6910 	work = 0;
6911 	if (test_bit(NAPI_STATE_SCHED, &n->state)) {
6912 		work = n->poll(n, weight);
6913 		trace_napi_poll(n, work, weight);
6914 	}
6915 
6916 	if (unlikely(work > weight))
6917 		pr_err_once("NAPI poll function %pS returned %d, exceeding its budget of %d.\n",
6918 			    n->poll, work, weight);
6919 
6920 	if (likely(work < weight))
6921 		return work;
6922 
6923 	/* Drivers must not modify the NAPI state if they
6924 	 * consume the entire weight.  In such cases this code
6925 	 * still "owns" the NAPI instance and therefore can
6926 	 * move the instance around on the list at-will.
6927 	 */
6928 	if (unlikely(napi_disable_pending(n))) {
6929 		napi_complete(n);
6930 		return work;
6931 	}
6932 
6933 	/* The NAPI context has more processing work, but busy-polling
6934 	 * is preferred. Exit early.
6935 	 */
6936 	if (napi_prefer_busy_poll(n)) {
6937 		if (napi_complete_done(n, work)) {
6938 			/* If timeout is not set, we need to make sure
6939 			 * that the NAPI is re-scheduled.
6940 			 */
6941 			napi_schedule(n);
6942 		}
6943 		return work;
6944 	}
6945 
6946 	if (n->gro_bitmask) {
6947 		/* flush too old packets
6948 		 * If HZ < 1000, flush all packets.
6949 		 */
6950 		napi_gro_flush(n, HZ >= 1000);
6951 	}
6952 
6953 	gro_normal_list(n);
6954 
6955 	/* Some drivers may have called napi_schedule
6956 	 * prior to exhausting their budget.
6957 	 */
6958 	if (unlikely(!list_empty(&n->poll_list))) {
6959 		pr_warn_once("%s: Budget exhausted after napi rescheduled\n",
6960 			     n->dev ? n->dev->name : "backlog");
6961 		return work;
6962 	}
6963 
6964 	*repoll = true;
6965 
6966 	return work;
6967 }
6968 
6969 static int napi_poll(struct napi_struct *n, struct list_head *repoll)
6970 {
6971 	bool do_repoll = false;
6972 	void *have;
6973 	int work;
6974 
6975 	list_del_init(&n->poll_list);
6976 
6977 	have = netpoll_poll_lock(n);
6978 
6979 	work = __napi_poll(n, &do_repoll);
6980 
6981 	if (do_repoll)
6982 		list_add_tail(&n->poll_list, repoll);
6983 
6984 	netpoll_poll_unlock(have);
6985 
6986 	return work;
6987 }
6988 
6989 static int napi_thread_wait(struct napi_struct *napi)
6990 {
6991 	bool woken = false;
6992 
6993 	set_current_state(TASK_INTERRUPTIBLE);
6994 
6995 	while (!kthread_should_stop() && !napi_disable_pending(napi)) {
6996 		/* Testing SCHED_THREADED bit here to make sure the current
6997 		 * kthread owns this napi and could poll on this napi.
6998 		 * Testing SCHED bit is not enough because SCHED bit might be
6999 		 * set by some other busy poll thread or by napi_disable().
7000 		 */
7001 		if (test_bit(NAPI_STATE_SCHED_THREADED, &napi->state) || woken) {
7002 			WARN_ON(!list_empty(&napi->poll_list));
7003 			__set_current_state(TASK_RUNNING);
7004 			return 0;
7005 		}
7006 
7007 		schedule();
7008 		/* woken being true indicates this thread owns this napi. */
7009 		woken = true;
7010 		set_current_state(TASK_INTERRUPTIBLE);
7011 	}
7012 	__set_current_state(TASK_RUNNING);
7013 	return -1;
7014 }
7015 
7016 static int napi_threaded_poll(void *data)
7017 {
7018 	struct napi_struct *napi = data;
7019 	void *have;
7020 
7021 	while (!napi_thread_wait(napi)) {
7022 		for (;;) {
7023 			bool repoll = false;
7024 
7025 			local_bh_disable();
7026 
7027 			have = netpoll_poll_lock(napi);
7028 			__napi_poll(napi, &repoll);
7029 			netpoll_poll_unlock(have);
7030 
7031 			local_bh_enable();
7032 
7033 			if (!repoll)
7034 				break;
7035 
7036 			cond_resched();
7037 		}
7038 	}
7039 	return 0;
7040 }
7041 
7042 static __latent_entropy void net_rx_action(struct softirq_action *h)
7043 {
7044 	struct softnet_data *sd = this_cpu_ptr(&softnet_data);
7045 	unsigned long time_limit = jiffies +
7046 		usecs_to_jiffies(netdev_budget_usecs);
7047 	int budget = netdev_budget;
7048 	LIST_HEAD(list);
7049 	LIST_HEAD(repoll);
7050 
7051 	local_irq_disable();
7052 	list_splice_init(&sd->poll_list, &list);
7053 	local_irq_enable();
7054 
7055 	for (;;) {
7056 		struct napi_struct *n;
7057 
7058 		if (list_empty(&list)) {
7059 			if (!sd_has_rps_ipi_waiting(sd) && list_empty(&repoll))
7060 				return;
7061 			break;
7062 		}
7063 
7064 		n = list_first_entry(&list, struct napi_struct, poll_list);
7065 		budget -= napi_poll(n, &repoll);
7066 
7067 		/* If softirq window is exhausted then punt.
7068 		 * Allow this to run for 2 jiffies since which will allow
7069 		 * an average latency of 1.5/HZ.
7070 		 */
7071 		if (unlikely(budget <= 0 ||
7072 			     time_after_eq(jiffies, time_limit))) {
7073 			sd->time_squeeze++;
7074 			break;
7075 		}
7076 	}
7077 
7078 	local_irq_disable();
7079 
7080 	list_splice_tail_init(&sd->poll_list, &list);
7081 	list_splice_tail(&repoll, &list);
7082 	list_splice(&list, &sd->poll_list);
7083 	if (!list_empty(&sd->poll_list))
7084 		__raise_softirq_irqoff(NET_RX_SOFTIRQ);
7085 
7086 	net_rps_action_and_irq_enable(sd);
7087 }
7088 
7089 struct netdev_adjacent {
7090 	struct net_device *dev;
7091 
7092 	/* upper master flag, there can only be one master device per list */
7093 	bool master;
7094 
7095 	/* lookup ignore flag */
7096 	bool ignore;
7097 
7098 	/* counter for the number of times this device was added to us */
7099 	u16 ref_nr;
7100 
7101 	/* private field for the users */
7102 	void *private;
7103 
7104 	struct list_head list;
7105 	struct rcu_head rcu;
7106 };
7107 
7108 static struct netdev_adjacent *__netdev_find_adj(struct net_device *adj_dev,
7109 						 struct list_head *adj_list)
7110 {
7111 	struct netdev_adjacent *adj;
7112 
7113 	list_for_each_entry(adj, adj_list, list) {
7114 		if (adj->dev == adj_dev)
7115 			return adj;
7116 	}
7117 	return NULL;
7118 }
7119 
7120 static int ____netdev_has_upper_dev(struct net_device *upper_dev,
7121 				    struct netdev_nested_priv *priv)
7122 {
7123 	struct net_device *dev = (struct net_device *)priv->data;
7124 
7125 	return upper_dev == dev;
7126 }
7127 
7128 /**
7129  * netdev_has_upper_dev - Check if device is linked to an upper device
7130  * @dev: device
7131  * @upper_dev: upper device to check
7132  *
7133  * Find out if a device is linked to specified upper device and return true
7134  * in case it is. Note that this checks only immediate upper device,
7135  * not through a complete stack of devices. The caller must hold the RTNL lock.
7136  */
7137 bool netdev_has_upper_dev(struct net_device *dev,
7138 			  struct net_device *upper_dev)
7139 {
7140 	struct netdev_nested_priv priv = {
7141 		.data = (void *)upper_dev,
7142 	};
7143 
7144 	ASSERT_RTNL();
7145 
7146 	return netdev_walk_all_upper_dev_rcu(dev, ____netdev_has_upper_dev,
7147 					     &priv);
7148 }
7149 EXPORT_SYMBOL(netdev_has_upper_dev);
7150 
7151 /**
7152  * netdev_has_upper_dev_all_rcu - Check if device is linked to an upper device
7153  * @dev: device
7154  * @upper_dev: upper device to check
7155  *
7156  * Find out if a device is linked to specified upper device and return true
7157  * in case it is. Note that this checks the entire upper device chain.
7158  * The caller must hold rcu lock.
7159  */
7160 
7161 bool netdev_has_upper_dev_all_rcu(struct net_device *dev,
7162 				  struct net_device *upper_dev)
7163 {
7164 	struct netdev_nested_priv priv = {
7165 		.data = (void *)upper_dev,
7166 	};
7167 
7168 	return !!netdev_walk_all_upper_dev_rcu(dev, ____netdev_has_upper_dev,
7169 					       &priv);
7170 }
7171 EXPORT_SYMBOL(netdev_has_upper_dev_all_rcu);
7172 
7173 /**
7174  * netdev_has_any_upper_dev - Check if device is linked to some device
7175  * @dev: device
7176  *
7177  * Find out if a device is linked to an upper device and return true in case
7178  * it is. The caller must hold the RTNL lock.
7179  */
7180 bool netdev_has_any_upper_dev(struct net_device *dev)
7181 {
7182 	ASSERT_RTNL();
7183 
7184 	return !list_empty(&dev->adj_list.upper);
7185 }
7186 EXPORT_SYMBOL(netdev_has_any_upper_dev);
7187 
7188 /**
7189  * netdev_master_upper_dev_get - Get master upper device
7190  * @dev: device
7191  *
7192  * Find a master upper device and return pointer to it or NULL in case
7193  * it's not there. The caller must hold the RTNL lock.
7194  */
7195 struct net_device *netdev_master_upper_dev_get(struct net_device *dev)
7196 {
7197 	struct netdev_adjacent *upper;
7198 
7199 	ASSERT_RTNL();
7200 
7201 	if (list_empty(&dev->adj_list.upper))
7202 		return NULL;
7203 
7204 	upper = list_first_entry(&dev->adj_list.upper,
7205 				 struct netdev_adjacent, list);
7206 	if (likely(upper->master))
7207 		return upper->dev;
7208 	return NULL;
7209 }
7210 EXPORT_SYMBOL(netdev_master_upper_dev_get);
7211 
7212 static struct net_device *__netdev_master_upper_dev_get(struct net_device *dev)
7213 {
7214 	struct netdev_adjacent *upper;
7215 
7216 	ASSERT_RTNL();
7217 
7218 	if (list_empty(&dev->adj_list.upper))
7219 		return NULL;
7220 
7221 	upper = list_first_entry(&dev->adj_list.upper,
7222 				 struct netdev_adjacent, list);
7223 	if (likely(upper->master) && !upper->ignore)
7224 		return upper->dev;
7225 	return NULL;
7226 }
7227 
7228 /**
7229  * netdev_has_any_lower_dev - Check if device is linked to some device
7230  * @dev: device
7231  *
7232  * Find out if a device is linked to a lower device and return true in case
7233  * it is. The caller must hold the RTNL lock.
7234  */
7235 static bool netdev_has_any_lower_dev(struct net_device *dev)
7236 {
7237 	ASSERT_RTNL();
7238 
7239 	return !list_empty(&dev->adj_list.lower);
7240 }
7241 
7242 void *netdev_adjacent_get_private(struct list_head *adj_list)
7243 {
7244 	struct netdev_adjacent *adj;
7245 
7246 	adj = list_entry(adj_list, struct netdev_adjacent, list);
7247 
7248 	return adj->private;
7249 }
7250 EXPORT_SYMBOL(netdev_adjacent_get_private);
7251 
7252 /**
7253  * netdev_upper_get_next_dev_rcu - Get the next dev from upper list
7254  * @dev: device
7255  * @iter: list_head ** of the current position
7256  *
7257  * Gets the next device from the dev's upper list, starting from iter
7258  * position. The caller must hold RCU read lock.
7259  */
7260 struct net_device *netdev_upper_get_next_dev_rcu(struct net_device *dev,
7261 						 struct list_head **iter)
7262 {
7263 	struct netdev_adjacent *upper;
7264 
7265 	WARN_ON_ONCE(!rcu_read_lock_held() && !lockdep_rtnl_is_held());
7266 
7267 	upper = list_entry_rcu((*iter)->next, struct netdev_adjacent, list);
7268 
7269 	if (&upper->list == &dev->adj_list.upper)
7270 		return NULL;
7271 
7272 	*iter = &upper->list;
7273 
7274 	return upper->dev;
7275 }
7276 EXPORT_SYMBOL(netdev_upper_get_next_dev_rcu);
7277 
7278 static struct net_device *__netdev_next_upper_dev(struct net_device *dev,
7279 						  struct list_head **iter,
7280 						  bool *ignore)
7281 {
7282 	struct netdev_adjacent *upper;
7283 
7284 	upper = list_entry((*iter)->next, struct netdev_adjacent, list);
7285 
7286 	if (&upper->list == &dev->adj_list.upper)
7287 		return NULL;
7288 
7289 	*iter = &upper->list;
7290 	*ignore = upper->ignore;
7291 
7292 	return upper->dev;
7293 }
7294 
7295 static struct net_device *netdev_next_upper_dev_rcu(struct net_device *dev,
7296 						    struct list_head **iter)
7297 {
7298 	struct netdev_adjacent *upper;
7299 
7300 	WARN_ON_ONCE(!rcu_read_lock_held() && !lockdep_rtnl_is_held());
7301 
7302 	upper = list_entry_rcu((*iter)->next, struct netdev_adjacent, list);
7303 
7304 	if (&upper->list == &dev->adj_list.upper)
7305 		return NULL;
7306 
7307 	*iter = &upper->list;
7308 
7309 	return upper->dev;
7310 }
7311 
7312 static int __netdev_walk_all_upper_dev(struct net_device *dev,
7313 				       int (*fn)(struct net_device *dev,
7314 					 struct netdev_nested_priv *priv),
7315 				       struct netdev_nested_priv *priv)
7316 {
7317 	struct net_device *udev, *next, *now, *dev_stack[MAX_NEST_DEV + 1];
7318 	struct list_head *niter, *iter, *iter_stack[MAX_NEST_DEV + 1];
7319 	int ret, cur = 0;
7320 	bool ignore;
7321 
7322 	now = dev;
7323 	iter = &dev->adj_list.upper;
7324 
7325 	while (1) {
7326 		if (now != dev) {
7327 			ret = fn(now, priv);
7328 			if (ret)
7329 				return ret;
7330 		}
7331 
7332 		next = NULL;
7333 		while (1) {
7334 			udev = __netdev_next_upper_dev(now, &iter, &ignore);
7335 			if (!udev)
7336 				break;
7337 			if (ignore)
7338 				continue;
7339 
7340 			next = udev;
7341 			niter = &udev->adj_list.upper;
7342 			dev_stack[cur] = now;
7343 			iter_stack[cur++] = iter;
7344 			break;
7345 		}
7346 
7347 		if (!next) {
7348 			if (!cur)
7349 				return 0;
7350 			next = dev_stack[--cur];
7351 			niter = iter_stack[cur];
7352 		}
7353 
7354 		now = next;
7355 		iter = niter;
7356 	}
7357 
7358 	return 0;
7359 }
7360 
7361 int netdev_walk_all_upper_dev_rcu(struct net_device *dev,
7362 				  int (*fn)(struct net_device *dev,
7363 					    struct netdev_nested_priv *priv),
7364 				  struct netdev_nested_priv *priv)
7365 {
7366 	struct net_device *udev, *next, *now, *dev_stack[MAX_NEST_DEV + 1];
7367 	struct list_head *niter, *iter, *iter_stack[MAX_NEST_DEV + 1];
7368 	int ret, cur = 0;
7369 
7370 	now = dev;
7371 	iter = &dev->adj_list.upper;
7372 
7373 	while (1) {
7374 		if (now != dev) {
7375 			ret = fn(now, priv);
7376 			if (ret)
7377 				return ret;
7378 		}
7379 
7380 		next = NULL;
7381 		while (1) {
7382 			udev = netdev_next_upper_dev_rcu(now, &iter);
7383 			if (!udev)
7384 				break;
7385 
7386 			next = udev;
7387 			niter = &udev->adj_list.upper;
7388 			dev_stack[cur] = now;
7389 			iter_stack[cur++] = iter;
7390 			break;
7391 		}
7392 
7393 		if (!next) {
7394 			if (!cur)
7395 				return 0;
7396 			next = dev_stack[--cur];
7397 			niter = iter_stack[cur];
7398 		}
7399 
7400 		now = next;
7401 		iter = niter;
7402 	}
7403 
7404 	return 0;
7405 }
7406 EXPORT_SYMBOL_GPL(netdev_walk_all_upper_dev_rcu);
7407 
7408 static bool __netdev_has_upper_dev(struct net_device *dev,
7409 				   struct net_device *upper_dev)
7410 {
7411 	struct netdev_nested_priv priv = {
7412 		.flags = 0,
7413 		.data = (void *)upper_dev,
7414 	};
7415 
7416 	ASSERT_RTNL();
7417 
7418 	return __netdev_walk_all_upper_dev(dev, ____netdev_has_upper_dev,
7419 					   &priv);
7420 }
7421 
7422 /**
7423  * netdev_lower_get_next_private - Get the next ->private from the
7424  *				   lower neighbour list
7425  * @dev: device
7426  * @iter: list_head ** of the current position
7427  *
7428  * Gets the next netdev_adjacent->private from the dev's lower neighbour
7429  * list, starting from iter position. The caller must hold either hold the
7430  * RTNL lock or its own locking that guarantees that the neighbour lower
7431  * list will remain unchanged.
7432  */
7433 void *netdev_lower_get_next_private(struct net_device *dev,
7434 				    struct list_head **iter)
7435 {
7436 	struct netdev_adjacent *lower;
7437 
7438 	lower = list_entry(*iter, struct netdev_adjacent, list);
7439 
7440 	if (&lower->list == &dev->adj_list.lower)
7441 		return NULL;
7442 
7443 	*iter = lower->list.next;
7444 
7445 	return lower->private;
7446 }
7447 EXPORT_SYMBOL(netdev_lower_get_next_private);
7448 
7449 /**
7450  * netdev_lower_get_next_private_rcu - Get the next ->private from the
7451  *				       lower neighbour list, RCU
7452  *				       variant
7453  * @dev: device
7454  * @iter: list_head ** of the current position
7455  *
7456  * Gets the next netdev_adjacent->private from the dev's lower neighbour
7457  * list, starting from iter position. The caller must hold RCU read lock.
7458  */
7459 void *netdev_lower_get_next_private_rcu(struct net_device *dev,
7460 					struct list_head **iter)
7461 {
7462 	struct netdev_adjacent *lower;
7463 
7464 	WARN_ON_ONCE(!rcu_read_lock_held());
7465 
7466 	lower = list_entry_rcu((*iter)->next, struct netdev_adjacent, list);
7467 
7468 	if (&lower->list == &dev->adj_list.lower)
7469 		return NULL;
7470 
7471 	*iter = &lower->list;
7472 
7473 	return lower->private;
7474 }
7475 EXPORT_SYMBOL(netdev_lower_get_next_private_rcu);
7476 
7477 /**
7478  * netdev_lower_get_next - Get the next device from the lower neighbour
7479  *                         list
7480  * @dev: device
7481  * @iter: list_head ** of the current position
7482  *
7483  * Gets the next netdev_adjacent from the dev's lower neighbour
7484  * list, starting from iter position. The caller must hold RTNL lock or
7485  * its own locking that guarantees that the neighbour lower
7486  * list will remain unchanged.
7487  */
7488 void *netdev_lower_get_next(struct net_device *dev, struct list_head **iter)
7489 {
7490 	struct netdev_adjacent *lower;
7491 
7492 	lower = list_entry(*iter, struct netdev_adjacent, list);
7493 
7494 	if (&lower->list == &dev->adj_list.lower)
7495 		return NULL;
7496 
7497 	*iter = lower->list.next;
7498 
7499 	return lower->dev;
7500 }
7501 EXPORT_SYMBOL(netdev_lower_get_next);
7502 
7503 static struct net_device *netdev_next_lower_dev(struct net_device *dev,
7504 						struct list_head **iter)
7505 {
7506 	struct netdev_adjacent *lower;
7507 
7508 	lower = list_entry((*iter)->next, struct netdev_adjacent, list);
7509 
7510 	if (&lower->list == &dev->adj_list.lower)
7511 		return NULL;
7512 
7513 	*iter = &lower->list;
7514 
7515 	return lower->dev;
7516 }
7517 
7518 static struct net_device *__netdev_next_lower_dev(struct net_device *dev,
7519 						  struct list_head **iter,
7520 						  bool *ignore)
7521 {
7522 	struct netdev_adjacent *lower;
7523 
7524 	lower = list_entry((*iter)->next, struct netdev_adjacent, list);
7525 
7526 	if (&lower->list == &dev->adj_list.lower)
7527 		return NULL;
7528 
7529 	*iter = &lower->list;
7530 	*ignore = lower->ignore;
7531 
7532 	return lower->dev;
7533 }
7534 
7535 int netdev_walk_all_lower_dev(struct net_device *dev,
7536 			      int (*fn)(struct net_device *dev,
7537 					struct netdev_nested_priv *priv),
7538 			      struct netdev_nested_priv *priv)
7539 {
7540 	struct net_device *ldev, *next, *now, *dev_stack[MAX_NEST_DEV + 1];
7541 	struct list_head *niter, *iter, *iter_stack[MAX_NEST_DEV + 1];
7542 	int ret, cur = 0;
7543 
7544 	now = dev;
7545 	iter = &dev->adj_list.lower;
7546 
7547 	while (1) {
7548 		if (now != dev) {
7549 			ret = fn(now, priv);
7550 			if (ret)
7551 				return ret;
7552 		}
7553 
7554 		next = NULL;
7555 		while (1) {
7556 			ldev = netdev_next_lower_dev(now, &iter);
7557 			if (!ldev)
7558 				break;
7559 
7560 			next = ldev;
7561 			niter = &ldev->adj_list.lower;
7562 			dev_stack[cur] = now;
7563 			iter_stack[cur++] = iter;
7564 			break;
7565 		}
7566 
7567 		if (!next) {
7568 			if (!cur)
7569 				return 0;
7570 			next = dev_stack[--cur];
7571 			niter = iter_stack[cur];
7572 		}
7573 
7574 		now = next;
7575 		iter = niter;
7576 	}
7577 
7578 	return 0;
7579 }
7580 EXPORT_SYMBOL_GPL(netdev_walk_all_lower_dev);
7581 
7582 static int __netdev_walk_all_lower_dev(struct net_device *dev,
7583 				       int (*fn)(struct net_device *dev,
7584 					 struct netdev_nested_priv *priv),
7585 				       struct netdev_nested_priv *priv)
7586 {
7587 	struct net_device *ldev, *next, *now, *dev_stack[MAX_NEST_DEV + 1];
7588 	struct list_head *niter, *iter, *iter_stack[MAX_NEST_DEV + 1];
7589 	int ret, cur = 0;
7590 	bool ignore;
7591 
7592 	now = dev;
7593 	iter = &dev->adj_list.lower;
7594 
7595 	while (1) {
7596 		if (now != dev) {
7597 			ret = fn(now, priv);
7598 			if (ret)
7599 				return ret;
7600 		}
7601 
7602 		next = NULL;
7603 		while (1) {
7604 			ldev = __netdev_next_lower_dev(now, &iter, &ignore);
7605 			if (!ldev)
7606 				break;
7607 			if (ignore)
7608 				continue;
7609 
7610 			next = ldev;
7611 			niter = &ldev->adj_list.lower;
7612 			dev_stack[cur] = now;
7613 			iter_stack[cur++] = iter;
7614 			break;
7615 		}
7616 
7617 		if (!next) {
7618 			if (!cur)
7619 				return 0;
7620 			next = dev_stack[--cur];
7621 			niter = iter_stack[cur];
7622 		}
7623 
7624 		now = next;
7625 		iter = niter;
7626 	}
7627 
7628 	return 0;
7629 }
7630 
7631 struct net_device *netdev_next_lower_dev_rcu(struct net_device *dev,
7632 					     struct list_head **iter)
7633 {
7634 	struct netdev_adjacent *lower;
7635 
7636 	lower = list_entry_rcu((*iter)->next, struct netdev_adjacent, list);
7637 	if (&lower->list == &dev->adj_list.lower)
7638 		return NULL;
7639 
7640 	*iter = &lower->list;
7641 
7642 	return lower->dev;
7643 }
7644 EXPORT_SYMBOL(netdev_next_lower_dev_rcu);
7645 
7646 static u8 __netdev_upper_depth(struct net_device *dev)
7647 {
7648 	struct net_device *udev;
7649 	struct list_head *iter;
7650 	u8 max_depth = 0;
7651 	bool ignore;
7652 
7653 	for (iter = &dev->adj_list.upper,
7654 	     udev = __netdev_next_upper_dev(dev, &iter, &ignore);
7655 	     udev;
7656 	     udev = __netdev_next_upper_dev(dev, &iter, &ignore)) {
7657 		if (ignore)
7658 			continue;
7659 		if (max_depth < udev->upper_level)
7660 			max_depth = udev->upper_level;
7661 	}
7662 
7663 	return max_depth;
7664 }
7665 
7666 static u8 __netdev_lower_depth(struct net_device *dev)
7667 {
7668 	struct net_device *ldev;
7669 	struct list_head *iter;
7670 	u8 max_depth = 0;
7671 	bool ignore;
7672 
7673 	for (iter = &dev->adj_list.lower,
7674 	     ldev = __netdev_next_lower_dev(dev, &iter, &ignore);
7675 	     ldev;
7676 	     ldev = __netdev_next_lower_dev(dev, &iter, &ignore)) {
7677 		if (ignore)
7678 			continue;
7679 		if (max_depth < ldev->lower_level)
7680 			max_depth = ldev->lower_level;
7681 	}
7682 
7683 	return max_depth;
7684 }
7685 
7686 static int __netdev_update_upper_level(struct net_device *dev,
7687 				       struct netdev_nested_priv *__unused)
7688 {
7689 	dev->upper_level = __netdev_upper_depth(dev) + 1;
7690 	return 0;
7691 }
7692 
7693 static int __netdev_update_lower_level(struct net_device *dev,
7694 				       struct netdev_nested_priv *priv)
7695 {
7696 	dev->lower_level = __netdev_lower_depth(dev) + 1;
7697 
7698 #ifdef CONFIG_LOCKDEP
7699 	if (!priv)
7700 		return 0;
7701 
7702 	if (priv->flags & NESTED_SYNC_IMM)
7703 		dev->nested_level = dev->lower_level - 1;
7704 	if (priv->flags & NESTED_SYNC_TODO)
7705 		net_unlink_todo(dev);
7706 #endif
7707 	return 0;
7708 }
7709 
7710 int netdev_walk_all_lower_dev_rcu(struct net_device *dev,
7711 				  int (*fn)(struct net_device *dev,
7712 					    struct netdev_nested_priv *priv),
7713 				  struct netdev_nested_priv *priv)
7714 {
7715 	struct net_device *ldev, *next, *now, *dev_stack[MAX_NEST_DEV + 1];
7716 	struct list_head *niter, *iter, *iter_stack[MAX_NEST_DEV + 1];
7717 	int ret, cur = 0;
7718 
7719 	now = dev;
7720 	iter = &dev->adj_list.lower;
7721 
7722 	while (1) {
7723 		if (now != dev) {
7724 			ret = fn(now, priv);
7725 			if (ret)
7726 				return ret;
7727 		}
7728 
7729 		next = NULL;
7730 		while (1) {
7731 			ldev = netdev_next_lower_dev_rcu(now, &iter);
7732 			if (!ldev)
7733 				break;
7734 
7735 			next = ldev;
7736 			niter = &ldev->adj_list.lower;
7737 			dev_stack[cur] = now;
7738 			iter_stack[cur++] = iter;
7739 			break;
7740 		}
7741 
7742 		if (!next) {
7743 			if (!cur)
7744 				return 0;
7745 			next = dev_stack[--cur];
7746 			niter = iter_stack[cur];
7747 		}
7748 
7749 		now = next;
7750 		iter = niter;
7751 	}
7752 
7753 	return 0;
7754 }
7755 EXPORT_SYMBOL_GPL(netdev_walk_all_lower_dev_rcu);
7756 
7757 /**
7758  * netdev_lower_get_first_private_rcu - Get the first ->private from the
7759  *				       lower neighbour list, RCU
7760  *				       variant
7761  * @dev: device
7762  *
7763  * Gets the first netdev_adjacent->private from the dev's lower neighbour
7764  * list. The caller must hold RCU read lock.
7765  */
7766 void *netdev_lower_get_first_private_rcu(struct net_device *dev)
7767 {
7768 	struct netdev_adjacent *lower;
7769 
7770 	lower = list_first_or_null_rcu(&dev->adj_list.lower,
7771 			struct netdev_adjacent, list);
7772 	if (lower)
7773 		return lower->private;
7774 	return NULL;
7775 }
7776 EXPORT_SYMBOL(netdev_lower_get_first_private_rcu);
7777 
7778 /**
7779  * netdev_master_upper_dev_get_rcu - Get master upper device
7780  * @dev: device
7781  *
7782  * Find a master upper device and return pointer to it or NULL in case
7783  * it's not there. The caller must hold the RCU read lock.
7784  */
7785 struct net_device *netdev_master_upper_dev_get_rcu(struct net_device *dev)
7786 {
7787 	struct netdev_adjacent *upper;
7788 
7789 	upper = list_first_or_null_rcu(&dev->adj_list.upper,
7790 				       struct netdev_adjacent, list);
7791 	if (upper && likely(upper->master))
7792 		return upper->dev;
7793 	return NULL;
7794 }
7795 EXPORT_SYMBOL(netdev_master_upper_dev_get_rcu);
7796 
7797 static int netdev_adjacent_sysfs_add(struct net_device *dev,
7798 			      struct net_device *adj_dev,
7799 			      struct list_head *dev_list)
7800 {
7801 	char linkname[IFNAMSIZ+7];
7802 
7803 	sprintf(linkname, dev_list == &dev->adj_list.upper ?
7804 		"upper_%s" : "lower_%s", adj_dev->name);
7805 	return sysfs_create_link(&(dev->dev.kobj), &(adj_dev->dev.kobj),
7806 				 linkname);
7807 }
7808 static void netdev_adjacent_sysfs_del(struct net_device *dev,
7809 			       char *name,
7810 			       struct list_head *dev_list)
7811 {
7812 	char linkname[IFNAMSIZ+7];
7813 
7814 	sprintf(linkname, dev_list == &dev->adj_list.upper ?
7815 		"upper_%s" : "lower_%s", name);
7816 	sysfs_remove_link(&(dev->dev.kobj), linkname);
7817 }
7818 
7819 static inline bool netdev_adjacent_is_neigh_list(struct net_device *dev,
7820 						 struct net_device *adj_dev,
7821 						 struct list_head *dev_list)
7822 {
7823 	return (dev_list == &dev->adj_list.upper ||
7824 		dev_list == &dev->adj_list.lower) &&
7825 		net_eq(dev_net(dev), dev_net(adj_dev));
7826 }
7827 
7828 static int __netdev_adjacent_dev_insert(struct net_device *dev,
7829 					struct net_device *adj_dev,
7830 					struct list_head *dev_list,
7831 					void *private, bool master)
7832 {
7833 	struct netdev_adjacent *adj;
7834 	int ret;
7835 
7836 	adj = __netdev_find_adj(adj_dev, dev_list);
7837 
7838 	if (adj) {
7839 		adj->ref_nr += 1;
7840 		pr_debug("Insert adjacency: dev %s adj_dev %s adj->ref_nr %d\n",
7841 			 dev->name, adj_dev->name, adj->ref_nr);
7842 
7843 		return 0;
7844 	}
7845 
7846 	adj = kmalloc(sizeof(*adj), GFP_KERNEL);
7847 	if (!adj)
7848 		return -ENOMEM;
7849 
7850 	adj->dev = adj_dev;
7851 	adj->master = master;
7852 	adj->ref_nr = 1;
7853 	adj->private = private;
7854 	adj->ignore = false;
7855 	dev_hold(adj_dev);
7856 
7857 	pr_debug("Insert adjacency: dev %s adj_dev %s adj->ref_nr %d; dev_hold on %s\n",
7858 		 dev->name, adj_dev->name, adj->ref_nr, adj_dev->name);
7859 
7860 	if (netdev_adjacent_is_neigh_list(dev, adj_dev, dev_list)) {
7861 		ret = netdev_adjacent_sysfs_add(dev, adj_dev, dev_list);
7862 		if (ret)
7863 			goto free_adj;
7864 	}
7865 
7866 	/* Ensure that master link is always the first item in list. */
7867 	if (master) {
7868 		ret = sysfs_create_link(&(dev->dev.kobj),
7869 					&(adj_dev->dev.kobj), "master");
7870 		if (ret)
7871 			goto remove_symlinks;
7872 
7873 		list_add_rcu(&adj->list, dev_list);
7874 	} else {
7875 		list_add_tail_rcu(&adj->list, dev_list);
7876 	}
7877 
7878 	return 0;
7879 
7880 remove_symlinks:
7881 	if (netdev_adjacent_is_neigh_list(dev, adj_dev, dev_list))
7882 		netdev_adjacent_sysfs_del(dev, adj_dev->name, dev_list);
7883 free_adj:
7884 	kfree(adj);
7885 	dev_put(adj_dev);
7886 
7887 	return ret;
7888 }
7889 
7890 static void __netdev_adjacent_dev_remove(struct net_device *dev,
7891 					 struct net_device *adj_dev,
7892 					 u16 ref_nr,
7893 					 struct list_head *dev_list)
7894 {
7895 	struct netdev_adjacent *adj;
7896 
7897 	pr_debug("Remove adjacency: dev %s adj_dev %s ref_nr %d\n",
7898 		 dev->name, adj_dev->name, ref_nr);
7899 
7900 	adj = __netdev_find_adj(adj_dev, dev_list);
7901 
7902 	if (!adj) {
7903 		pr_err("Adjacency does not exist for device %s from %s\n",
7904 		       dev->name, adj_dev->name);
7905 		WARN_ON(1);
7906 		return;
7907 	}
7908 
7909 	if (adj->ref_nr > ref_nr) {
7910 		pr_debug("adjacency: %s to %s ref_nr - %d = %d\n",
7911 			 dev->name, adj_dev->name, ref_nr,
7912 			 adj->ref_nr - ref_nr);
7913 		adj->ref_nr -= ref_nr;
7914 		return;
7915 	}
7916 
7917 	if (adj->master)
7918 		sysfs_remove_link(&(dev->dev.kobj), "master");
7919 
7920 	if (netdev_adjacent_is_neigh_list(dev, adj_dev, dev_list))
7921 		netdev_adjacent_sysfs_del(dev, adj_dev->name, dev_list);
7922 
7923 	list_del_rcu(&adj->list);
7924 	pr_debug("adjacency: dev_put for %s, because link removed from %s to %s\n",
7925 		 adj_dev->name, dev->name, adj_dev->name);
7926 	dev_put(adj_dev);
7927 	kfree_rcu(adj, rcu);
7928 }
7929 
7930 static int __netdev_adjacent_dev_link_lists(struct net_device *dev,
7931 					    struct net_device *upper_dev,
7932 					    struct list_head *up_list,
7933 					    struct list_head *down_list,
7934 					    void *private, bool master)
7935 {
7936 	int ret;
7937 
7938 	ret = __netdev_adjacent_dev_insert(dev, upper_dev, up_list,
7939 					   private, master);
7940 	if (ret)
7941 		return ret;
7942 
7943 	ret = __netdev_adjacent_dev_insert(upper_dev, dev, down_list,
7944 					   private, false);
7945 	if (ret) {
7946 		__netdev_adjacent_dev_remove(dev, upper_dev, 1, up_list);
7947 		return ret;
7948 	}
7949 
7950 	return 0;
7951 }
7952 
7953 static void __netdev_adjacent_dev_unlink_lists(struct net_device *dev,
7954 					       struct net_device *upper_dev,
7955 					       u16 ref_nr,
7956 					       struct list_head *up_list,
7957 					       struct list_head *down_list)
7958 {
7959 	__netdev_adjacent_dev_remove(dev, upper_dev, ref_nr, up_list);
7960 	__netdev_adjacent_dev_remove(upper_dev, dev, ref_nr, down_list);
7961 }
7962 
7963 static int __netdev_adjacent_dev_link_neighbour(struct net_device *dev,
7964 						struct net_device *upper_dev,
7965 						void *private, bool master)
7966 {
7967 	return __netdev_adjacent_dev_link_lists(dev, upper_dev,
7968 						&dev->adj_list.upper,
7969 						&upper_dev->adj_list.lower,
7970 						private, master);
7971 }
7972 
7973 static void __netdev_adjacent_dev_unlink_neighbour(struct net_device *dev,
7974 						   struct net_device *upper_dev)
7975 {
7976 	__netdev_adjacent_dev_unlink_lists(dev, upper_dev, 1,
7977 					   &dev->adj_list.upper,
7978 					   &upper_dev->adj_list.lower);
7979 }
7980 
7981 static int __netdev_upper_dev_link(struct net_device *dev,
7982 				   struct net_device *upper_dev, bool master,
7983 				   void *upper_priv, void *upper_info,
7984 				   struct netdev_nested_priv *priv,
7985 				   struct netlink_ext_ack *extack)
7986 {
7987 	struct netdev_notifier_changeupper_info changeupper_info = {
7988 		.info = {
7989 			.dev = dev,
7990 			.extack = extack,
7991 		},
7992 		.upper_dev = upper_dev,
7993 		.master = master,
7994 		.linking = true,
7995 		.upper_info = upper_info,
7996 	};
7997 	struct net_device *master_dev;
7998 	int ret = 0;
7999 
8000 	ASSERT_RTNL();
8001 
8002 	if (dev == upper_dev)
8003 		return -EBUSY;
8004 
8005 	/* To prevent loops, check if dev is not upper device to upper_dev. */
8006 	if (__netdev_has_upper_dev(upper_dev, dev))
8007 		return -EBUSY;
8008 
8009 	if ((dev->lower_level + upper_dev->upper_level) > MAX_NEST_DEV)
8010 		return -EMLINK;
8011 
8012 	if (!master) {
8013 		if (__netdev_has_upper_dev(dev, upper_dev))
8014 			return -EEXIST;
8015 	} else {
8016 		master_dev = __netdev_master_upper_dev_get(dev);
8017 		if (master_dev)
8018 			return master_dev == upper_dev ? -EEXIST : -EBUSY;
8019 	}
8020 
8021 	ret = call_netdevice_notifiers_info(NETDEV_PRECHANGEUPPER,
8022 					    &changeupper_info.info);
8023 	ret = notifier_to_errno(ret);
8024 	if (ret)
8025 		return ret;
8026 
8027 	ret = __netdev_adjacent_dev_link_neighbour(dev, upper_dev, upper_priv,
8028 						   master);
8029 	if (ret)
8030 		return ret;
8031 
8032 	ret = call_netdevice_notifiers_info(NETDEV_CHANGEUPPER,
8033 					    &changeupper_info.info);
8034 	ret = notifier_to_errno(ret);
8035 	if (ret)
8036 		goto rollback;
8037 
8038 	__netdev_update_upper_level(dev, NULL);
8039 	__netdev_walk_all_lower_dev(dev, __netdev_update_upper_level, NULL);
8040 
8041 	__netdev_update_lower_level(upper_dev, priv);
8042 	__netdev_walk_all_upper_dev(upper_dev, __netdev_update_lower_level,
8043 				    priv);
8044 
8045 	return 0;
8046 
8047 rollback:
8048 	__netdev_adjacent_dev_unlink_neighbour(dev, upper_dev);
8049 
8050 	return ret;
8051 }
8052 
8053 /**
8054  * netdev_upper_dev_link - Add a link to the upper device
8055  * @dev: device
8056  * @upper_dev: new upper device
8057  * @extack: netlink extended ack
8058  *
8059  * Adds a link to device which is upper to this one. The caller must hold
8060  * the RTNL lock. On a failure a negative errno code is returned.
8061  * On success the reference counts are adjusted and the function
8062  * returns zero.
8063  */
8064 int netdev_upper_dev_link(struct net_device *dev,
8065 			  struct net_device *upper_dev,
8066 			  struct netlink_ext_ack *extack)
8067 {
8068 	struct netdev_nested_priv priv = {
8069 		.flags = NESTED_SYNC_IMM | NESTED_SYNC_TODO,
8070 		.data = NULL,
8071 	};
8072 
8073 	return __netdev_upper_dev_link(dev, upper_dev, false,
8074 				       NULL, NULL, &priv, extack);
8075 }
8076 EXPORT_SYMBOL(netdev_upper_dev_link);
8077 
8078 /**
8079  * netdev_master_upper_dev_link - Add a master link to the upper device
8080  * @dev: device
8081  * @upper_dev: new upper device
8082  * @upper_priv: upper device private
8083  * @upper_info: upper info to be passed down via notifier
8084  * @extack: netlink extended ack
8085  *
8086  * Adds a link to device which is upper to this one. In this case, only
8087  * one master upper device can be linked, although other non-master devices
8088  * might be linked as well. The caller must hold the RTNL lock.
8089  * On a failure a negative errno code is returned. On success the reference
8090  * counts are adjusted and the function returns zero.
8091  */
8092 int netdev_master_upper_dev_link(struct net_device *dev,
8093 				 struct net_device *upper_dev,
8094 				 void *upper_priv, void *upper_info,
8095 				 struct netlink_ext_ack *extack)
8096 {
8097 	struct netdev_nested_priv priv = {
8098 		.flags = NESTED_SYNC_IMM | NESTED_SYNC_TODO,
8099 		.data = NULL,
8100 	};
8101 
8102 	return __netdev_upper_dev_link(dev, upper_dev, true,
8103 				       upper_priv, upper_info, &priv, extack);
8104 }
8105 EXPORT_SYMBOL(netdev_master_upper_dev_link);
8106 
8107 static void __netdev_upper_dev_unlink(struct net_device *dev,
8108 				      struct net_device *upper_dev,
8109 				      struct netdev_nested_priv *priv)
8110 {
8111 	struct netdev_notifier_changeupper_info changeupper_info = {
8112 		.info = {
8113 			.dev = dev,
8114 		},
8115 		.upper_dev = upper_dev,
8116 		.linking = false,
8117 	};
8118 
8119 	ASSERT_RTNL();
8120 
8121 	changeupper_info.master = netdev_master_upper_dev_get(dev) == upper_dev;
8122 
8123 	call_netdevice_notifiers_info(NETDEV_PRECHANGEUPPER,
8124 				      &changeupper_info.info);
8125 
8126 	__netdev_adjacent_dev_unlink_neighbour(dev, upper_dev);
8127 
8128 	call_netdevice_notifiers_info(NETDEV_CHANGEUPPER,
8129 				      &changeupper_info.info);
8130 
8131 	__netdev_update_upper_level(dev, NULL);
8132 	__netdev_walk_all_lower_dev(dev, __netdev_update_upper_level, NULL);
8133 
8134 	__netdev_update_lower_level(upper_dev, priv);
8135 	__netdev_walk_all_upper_dev(upper_dev, __netdev_update_lower_level,
8136 				    priv);
8137 }
8138 
8139 /**
8140  * netdev_upper_dev_unlink - Removes a link to upper device
8141  * @dev: device
8142  * @upper_dev: new upper device
8143  *
8144  * Removes a link to device which is upper to this one. The caller must hold
8145  * the RTNL lock.
8146  */
8147 void netdev_upper_dev_unlink(struct net_device *dev,
8148 			     struct net_device *upper_dev)
8149 {
8150 	struct netdev_nested_priv priv = {
8151 		.flags = NESTED_SYNC_TODO,
8152 		.data = NULL,
8153 	};
8154 
8155 	__netdev_upper_dev_unlink(dev, upper_dev, &priv);
8156 }
8157 EXPORT_SYMBOL(netdev_upper_dev_unlink);
8158 
8159 static void __netdev_adjacent_dev_set(struct net_device *upper_dev,
8160 				      struct net_device *lower_dev,
8161 				      bool val)
8162 {
8163 	struct netdev_adjacent *adj;
8164 
8165 	adj = __netdev_find_adj(lower_dev, &upper_dev->adj_list.lower);
8166 	if (adj)
8167 		adj->ignore = val;
8168 
8169 	adj = __netdev_find_adj(upper_dev, &lower_dev->adj_list.upper);
8170 	if (adj)
8171 		adj->ignore = val;
8172 }
8173 
8174 static void netdev_adjacent_dev_disable(struct net_device *upper_dev,
8175 					struct net_device *lower_dev)
8176 {
8177 	__netdev_adjacent_dev_set(upper_dev, lower_dev, true);
8178 }
8179 
8180 static void netdev_adjacent_dev_enable(struct net_device *upper_dev,
8181 				       struct net_device *lower_dev)
8182 {
8183 	__netdev_adjacent_dev_set(upper_dev, lower_dev, false);
8184 }
8185 
8186 int netdev_adjacent_change_prepare(struct net_device *old_dev,
8187 				   struct net_device *new_dev,
8188 				   struct net_device *dev,
8189 				   struct netlink_ext_ack *extack)
8190 {
8191 	struct netdev_nested_priv priv = {
8192 		.flags = 0,
8193 		.data = NULL,
8194 	};
8195 	int err;
8196 
8197 	if (!new_dev)
8198 		return 0;
8199 
8200 	if (old_dev && new_dev != old_dev)
8201 		netdev_adjacent_dev_disable(dev, old_dev);
8202 	err = __netdev_upper_dev_link(new_dev, dev, false, NULL, NULL, &priv,
8203 				      extack);
8204 	if (err) {
8205 		if (old_dev && new_dev != old_dev)
8206 			netdev_adjacent_dev_enable(dev, old_dev);
8207 		return err;
8208 	}
8209 
8210 	return 0;
8211 }
8212 EXPORT_SYMBOL(netdev_adjacent_change_prepare);
8213 
8214 void netdev_adjacent_change_commit(struct net_device *old_dev,
8215 				   struct net_device *new_dev,
8216 				   struct net_device *dev)
8217 {
8218 	struct netdev_nested_priv priv = {
8219 		.flags = NESTED_SYNC_IMM | NESTED_SYNC_TODO,
8220 		.data = NULL,
8221 	};
8222 
8223 	if (!new_dev || !old_dev)
8224 		return;
8225 
8226 	if (new_dev == old_dev)
8227 		return;
8228 
8229 	netdev_adjacent_dev_enable(dev, old_dev);
8230 	__netdev_upper_dev_unlink(old_dev, dev, &priv);
8231 }
8232 EXPORT_SYMBOL(netdev_adjacent_change_commit);
8233 
8234 void netdev_adjacent_change_abort(struct net_device *old_dev,
8235 				  struct net_device *new_dev,
8236 				  struct net_device *dev)
8237 {
8238 	struct netdev_nested_priv priv = {
8239 		.flags = 0,
8240 		.data = NULL,
8241 	};
8242 
8243 	if (!new_dev)
8244 		return;
8245 
8246 	if (old_dev && new_dev != old_dev)
8247 		netdev_adjacent_dev_enable(dev, old_dev);
8248 
8249 	__netdev_upper_dev_unlink(new_dev, dev, &priv);
8250 }
8251 EXPORT_SYMBOL(netdev_adjacent_change_abort);
8252 
8253 /**
8254  * netdev_bonding_info_change - Dispatch event about slave change
8255  * @dev: device
8256  * @bonding_info: info to dispatch
8257  *
8258  * Send NETDEV_BONDING_INFO to netdev notifiers with info.
8259  * The caller must hold the RTNL lock.
8260  */
8261 void netdev_bonding_info_change(struct net_device *dev,
8262 				struct netdev_bonding_info *bonding_info)
8263 {
8264 	struct netdev_notifier_bonding_info info = {
8265 		.info.dev = dev,
8266 	};
8267 
8268 	memcpy(&info.bonding_info, bonding_info,
8269 	       sizeof(struct netdev_bonding_info));
8270 	call_netdevice_notifiers_info(NETDEV_BONDING_INFO,
8271 				      &info.info);
8272 }
8273 EXPORT_SYMBOL(netdev_bonding_info_change);
8274 
8275 /**
8276  * netdev_get_xmit_slave - Get the xmit slave of master device
8277  * @dev: device
8278  * @skb: The packet
8279  * @all_slaves: assume all the slaves are active
8280  *
8281  * The reference counters are not incremented so the caller must be
8282  * careful with locks. The caller must hold RCU lock.
8283  * %NULL is returned if no slave is found.
8284  */
8285 
8286 struct net_device *netdev_get_xmit_slave(struct net_device *dev,
8287 					 struct sk_buff *skb,
8288 					 bool all_slaves)
8289 {
8290 	const struct net_device_ops *ops = dev->netdev_ops;
8291 
8292 	if (!ops->ndo_get_xmit_slave)
8293 		return NULL;
8294 	return ops->ndo_get_xmit_slave(dev, skb, all_slaves);
8295 }
8296 EXPORT_SYMBOL(netdev_get_xmit_slave);
8297 
8298 static struct net_device *netdev_sk_get_lower_dev(struct net_device *dev,
8299 						  struct sock *sk)
8300 {
8301 	const struct net_device_ops *ops = dev->netdev_ops;
8302 
8303 	if (!ops->ndo_sk_get_lower_dev)
8304 		return NULL;
8305 	return ops->ndo_sk_get_lower_dev(dev, sk);
8306 }
8307 
8308 /**
8309  * netdev_sk_get_lowest_dev - Get the lowest device in chain given device and socket
8310  * @dev: device
8311  * @sk: the socket
8312  *
8313  * %NULL is returned if no lower device is found.
8314  */
8315 
8316 struct net_device *netdev_sk_get_lowest_dev(struct net_device *dev,
8317 					    struct sock *sk)
8318 {
8319 	struct net_device *lower;
8320 
8321 	lower = netdev_sk_get_lower_dev(dev, sk);
8322 	while (lower) {
8323 		dev = lower;
8324 		lower = netdev_sk_get_lower_dev(dev, sk);
8325 	}
8326 
8327 	return dev;
8328 }
8329 EXPORT_SYMBOL(netdev_sk_get_lowest_dev);
8330 
8331 static void netdev_adjacent_add_links(struct net_device *dev)
8332 {
8333 	struct netdev_adjacent *iter;
8334 
8335 	struct net *net = dev_net(dev);
8336 
8337 	list_for_each_entry(iter, &dev->adj_list.upper, list) {
8338 		if (!net_eq(net, dev_net(iter->dev)))
8339 			continue;
8340 		netdev_adjacent_sysfs_add(iter->dev, dev,
8341 					  &iter->dev->adj_list.lower);
8342 		netdev_adjacent_sysfs_add(dev, iter->dev,
8343 					  &dev->adj_list.upper);
8344 	}
8345 
8346 	list_for_each_entry(iter, &dev->adj_list.lower, list) {
8347 		if (!net_eq(net, dev_net(iter->dev)))
8348 			continue;
8349 		netdev_adjacent_sysfs_add(iter->dev, dev,
8350 					  &iter->dev->adj_list.upper);
8351 		netdev_adjacent_sysfs_add(dev, iter->dev,
8352 					  &dev->adj_list.lower);
8353 	}
8354 }
8355 
8356 static void netdev_adjacent_del_links(struct net_device *dev)
8357 {
8358 	struct netdev_adjacent *iter;
8359 
8360 	struct net *net = dev_net(dev);
8361 
8362 	list_for_each_entry(iter, &dev->adj_list.upper, list) {
8363 		if (!net_eq(net, dev_net(iter->dev)))
8364 			continue;
8365 		netdev_adjacent_sysfs_del(iter->dev, dev->name,
8366 					  &iter->dev->adj_list.lower);
8367 		netdev_adjacent_sysfs_del(dev, iter->dev->name,
8368 					  &dev->adj_list.upper);
8369 	}
8370 
8371 	list_for_each_entry(iter, &dev->adj_list.lower, list) {
8372 		if (!net_eq(net, dev_net(iter->dev)))
8373 			continue;
8374 		netdev_adjacent_sysfs_del(iter->dev, dev->name,
8375 					  &iter->dev->adj_list.upper);
8376 		netdev_adjacent_sysfs_del(dev, iter->dev->name,
8377 					  &dev->adj_list.lower);
8378 	}
8379 }
8380 
8381 void netdev_adjacent_rename_links(struct net_device *dev, char *oldname)
8382 {
8383 	struct netdev_adjacent *iter;
8384 
8385 	struct net *net = dev_net(dev);
8386 
8387 	list_for_each_entry(iter, &dev->adj_list.upper, list) {
8388 		if (!net_eq(net, dev_net(iter->dev)))
8389 			continue;
8390 		netdev_adjacent_sysfs_del(iter->dev, oldname,
8391 					  &iter->dev->adj_list.lower);
8392 		netdev_adjacent_sysfs_add(iter->dev, dev,
8393 					  &iter->dev->adj_list.lower);
8394 	}
8395 
8396 	list_for_each_entry(iter, &dev->adj_list.lower, list) {
8397 		if (!net_eq(net, dev_net(iter->dev)))
8398 			continue;
8399 		netdev_adjacent_sysfs_del(iter->dev, oldname,
8400 					  &iter->dev->adj_list.upper);
8401 		netdev_adjacent_sysfs_add(iter->dev, dev,
8402 					  &iter->dev->adj_list.upper);
8403 	}
8404 }
8405 
8406 void *netdev_lower_dev_get_private(struct net_device *dev,
8407 				   struct net_device *lower_dev)
8408 {
8409 	struct netdev_adjacent *lower;
8410 
8411 	if (!lower_dev)
8412 		return NULL;
8413 	lower = __netdev_find_adj(lower_dev, &dev->adj_list.lower);
8414 	if (!lower)
8415 		return NULL;
8416 
8417 	return lower->private;
8418 }
8419 EXPORT_SYMBOL(netdev_lower_dev_get_private);
8420 
8421 
8422 /**
8423  * netdev_lower_state_changed - Dispatch event about lower device state change
8424  * @lower_dev: device
8425  * @lower_state_info: state to dispatch
8426  *
8427  * Send NETDEV_CHANGELOWERSTATE to netdev notifiers with info.
8428  * The caller must hold the RTNL lock.
8429  */
8430 void netdev_lower_state_changed(struct net_device *lower_dev,
8431 				void *lower_state_info)
8432 {
8433 	struct netdev_notifier_changelowerstate_info changelowerstate_info = {
8434 		.info.dev = lower_dev,
8435 	};
8436 
8437 	ASSERT_RTNL();
8438 	changelowerstate_info.lower_state_info = lower_state_info;
8439 	call_netdevice_notifiers_info(NETDEV_CHANGELOWERSTATE,
8440 				      &changelowerstate_info.info);
8441 }
8442 EXPORT_SYMBOL(netdev_lower_state_changed);
8443 
8444 static void dev_change_rx_flags(struct net_device *dev, int flags)
8445 {
8446 	const struct net_device_ops *ops = dev->netdev_ops;
8447 
8448 	if (ops->ndo_change_rx_flags)
8449 		ops->ndo_change_rx_flags(dev, flags);
8450 }
8451 
8452 static int __dev_set_promiscuity(struct net_device *dev, int inc, bool notify)
8453 {
8454 	unsigned int old_flags = dev->flags;
8455 	kuid_t uid;
8456 	kgid_t gid;
8457 
8458 	ASSERT_RTNL();
8459 
8460 	dev->flags |= IFF_PROMISC;
8461 	dev->promiscuity += inc;
8462 	if (dev->promiscuity == 0) {
8463 		/*
8464 		 * Avoid overflow.
8465 		 * If inc causes overflow, untouch promisc and return error.
8466 		 */
8467 		if (inc < 0)
8468 			dev->flags &= ~IFF_PROMISC;
8469 		else {
8470 			dev->promiscuity -= inc;
8471 			pr_warn("%s: promiscuity touches roof, set promiscuity failed. promiscuity feature of device might be broken.\n",
8472 				dev->name);
8473 			return -EOVERFLOW;
8474 		}
8475 	}
8476 	if (dev->flags != old_flags) {
8477 		pr_info("device %s %s promiscuous mode\n",
8478 			dev->name,
8479 			dev->flags & IFF_PROMISC ? "entered" : "left");
8480 		if (audit_enabled) {
8481 			current_uid_gid(&uid, &gid);
8482 			audit_log(audit_context(), GFP_ATOMIC,
8483 				  AUDIT_ANOM_PROMISCUOUS,
8484 				  "dev=%s prom=%d old_prom=%d auid=%u uid=%u gid=%u ses=%u",
8485 				  dev->name, (dev->flags & IFF_PROMISC),
8486 				  (old_flags & IFF_PROMISC),
8487 				  from_kuid(&init_user_ns, audit_get_loginuid(current)),
8488 				  from_kuid(&init_user_ns, uid),
8489 				  from_kgid(&init_user_ns, gid),
8490 				  audit_get_sessionid(current));
8491 		}
8492 
8493 		dev_change_rx_flags(dev, IFF_PROMISC);
8494 	}
8495 	if (notify)
8496 		__dev_notify_flags(dev, old_flags, IFF_PROMISC);
8497 	return 0;
8498 }
8499 
8500 /**
8501  *	dev_set_promiscuity	- update promiscuity count on a device
8502  *	@dev: device
8503  *	@inc: modifier
8504  *
8505  *	Add or remove promiscuity from a device. While the count in the device
8506  *	remains above zero the interface remains promiscuous. Once it hits zero
8507  *	the device reverts back to normal filtering operation. A negative inc
8508  *	value is used to drop promiscuity on the device.
8509  *	Return 0 if successful or a negative errno code on error.
8510  */
8511 int dev_set_promiscuity(struct net_device *dev, int inc)
8512 {
8513 	unsigned int old_flags = dev->flags;
8514 	int err;
8515 
8516 	err = __dev_set_promiscuity(dev, inc, true);
8517 	if (err < 0)
8518 		return err;
8519 	if (dev->flags != old_flags)
8520 		dev_set_rx_mode(dev);
8521 	return err;
8522 }
8523 EXPORT_SYMBOL(dev_set_promiscuity);
8524 
8525 static int __dev_set_allmulti(struct net_device *dev, int inc, bool notify)
8526 {
8527 	unsigned int old_flags = dev->flags, old_gflags = dev->gflags;
8528 
8529 	ASSERT_RTNL();
8530 
8531 	dev->flags |= IFF_ALLMULTI;
8532 	dev->allmulti += inc;
8533 	if (dev->allmulti == 0) {
8534 		/*
8535 		 * Avoid overflow.
8536 		 * If inc causes overflow, untouch allmulti and return error.
8537 		 */
8538 		if (inc < 0)
8539 			dev->flags &= ~IFF_ALLMULTI;
8540 		else {
8541 			dev->allmulti -= inc;
8542 			pr_warn("%s: allmulti touches roof, set allmulti failed. allmulti feature of device might be broken.\n",
8543 				dev->name);
8544 			return -EOVERFLOW;
8545 		}
8546 	}
8547 	if (dev->flags ^ old_flags) {
8548 		dev_change_rx_flags(dev, IFF_ALLMULTI);
8549 		dev_set_rx_mode(dev);
8550 		if (notify)
8551 			__dev_notify_flags(dev, old_flags,
8552 					   dev->gflags ^ old_gflags);
8553 	}
8554 	return 0;
8555 }
8556 
8557 /**
8558  *	dev_set_allmulti	- update allmulti count on a device
8559  *	@dev: device
8560  *	@inc: modifier
8561  *
8562  *	Add or remove reception of all multicast frames to a device. While the
8563  *	count in the device remains above zero the interface remains listening
8564  *	to all interfaces. Once it hits zero the device reverts back to normal
8565  *	filtering operation. A negative @inc value is used to drop the counter
8566  *	when releasing a resource needing all multicasts.
8567  *	Return 0 if successful or a negative errno code on error.
8568  */
8569 
8570 int dev_set_allmulti(struct net_device *dev, int inc)
8571 {
8572 	return __dev_set_allmulti(dev, inc, true);
8573 }
8574 EXPORT_SYMBOL(dev_set_allmulti);
8575 
8576 /*
8577  *	Upload unicast and multicast address lists to device and
8578  *	configure RX filtering. When the device doesn't support unicast
8579  *	filtering it is put in promiscuous mode while unicast addresses
8580  *	are present.
8581  */
8582 void __dev_set_rx_mode(struct net_device *dev)
8583 {
8584 	const struct net_device_ops *ops = dev->netdev_ops;
8585 
8586 	/* dev_open will call this function so the list will stay sane. */
8587 	if (!(dev->flags&IFF_UP))
8588 		return;
8589 
8590 	if (!netif_device_present(dev))
8591 		return;
8592 
8593 	if (!(dev->priv_flags & IFF_UNICAST_FLT)) {
8594 		/* Unicast addresses changes may only happen under the rtnl,
8595 		 * therefore calling __dev_set_promiscuity here is safe.
8596 		 */
8597 		if (!netdev_uc_empty(dev) && !dev->uc_promisc) {
8598 			__dev_set_promiscuity(dev, 1, false);
8599 			dev->uc_promisc = true;
8600 		} else if (netdev_uc_empty(dev) && dev->uc_promisc) {
8601 			__dev_set_promiscuity(dev, -1, false);
8602 			dev->uc_promisc = false;
8603 		}
8604 	}
8605 
8606 	if (ops->ndo_set_rx_mode)
8607 		ops->ndo_set_rx_mode(dev);
8608 }
8609 
8610 void dev_set_rx_mode(struct net_device *dev)
8611 {
8612 	netif_addr_lock_bh(dev);
8613 	__dev_set_rx_mode(dev);
8614 	netif_addr_unlock_bh(dev);
8615 }
8616 
8617 /**
8618  *	dev_get_flags - get flags reported to userspace
8619  *	@dev: device
8620  *
8621  *	Get the combination of flag bits exported through APIs to userspace.
8622  */
8623 unsigned int dev_get_flags(const struct net_device *dev)
8624 {
8625 	unsigned int flags;
8626 
8627 	flags = (dev->flags & ~(IFF_PROMISC |
8628 				IFF_ALLMULTI |
8629 				IFF_RUNNING |
8630 				IFF_LOWER_UP |
8631 				IFF_DORMANT)) |
8632 		(dev->gflags & (IFF_PROMISC |
8633 				IFF_ALLMULTI));
8634 
8635 	if (netif_running(dev)) {
8636 		if (netif_oper_up(dev))
8637 			flags |= IFF_RUNNING;
8638 		if (netif_carrier_ok(dev))
8639 			flags |= IFF_LOWER_UP;
8640 		if (netif_dormant(dev))
8641 			flags |= IFF_DORMANT;
8642 	}
8643 
8644 	return flags;
8645 }
8646 EXPORT_SYMBOL(dev_get_flags);
8647 
8648 int __dev_change_flags(struct net_device *dev, unsigned int flags,
8649 		       struct netlink_ext_ack *extack)
8650 {
8651 	unsigned int old_flags = dev->flags;
8652 	int ret;
8653 
8654 	ASSERT_RTNL();
8655 
8656 	/*
8657 	 *	Set the flags on our device.
8658 	 */
8659 
8660 	dev->flags = (flags & (IFF_DEBUG | IFF_NOTRAILERS | IFF_NOARP |
8661 			       IFF_DYNAMIC | IFF_MULTICAST | IFF_PORTSEL |
8662 			       IFF_AUTOMEDIA)) |
8663 		     (dev->flags & (IFF_UP | IFF_VOLATILE | IFF_PROMISC |
8664 				    IFF_ALLMULTI));
8665 
8666 	/*
8667 	 *	Load in the correct multicast list now the flags have changed.
8668 	 */
8669 
8670 	if ((old_flags ^ flags) & IFF_MULTICAST)
8671 		dev_change_rx_flags(dev, IFF_MULTICAST);
8672 
8673 	dev_set_rx_mode(dev);
8674 
8675 	/*
8676 	 *	Have we downed the interface. We handle IFF_UP ourselves
8677 	 *	according to user attempts to set it, rather than blindly
8678 	 *	setting it.
8679 	 */
8680 
8681 	ret = 0;
8682 	if ((old_flags ^ flags) & IFF_UP) {
8683 		if (old_flags & IFF_UP)
8684 			__dev_close(dev);
8685 		else
8686 			ret = __dev_open(dev, extack);
8687 	}
8688 
8689 	if ((flags ^ dev->gflags) & IFF_PROMISC) {
8690 		int inc = (flags & IFF_PROMISC) ? 1 : -1;
8691 		unsigned int old_flags = dev->flags;
8692 
8693 		dev->gflags ^= IFF_PROMISC;
8694 
8695 		if (__dev_set_promiscuity(dev, inc, false) >= 0)
8696 			if (dev->flags != old_flags)
8697 				dev_set_rx_mode(dev);
8698 	}
8699 
8700 	/* NOTE: order of synchronization of IFF_PROMISC and IFF_ALLMULTI
8701 	 * is important. Some (broken) drivers set IFF_PROMISC, when
8702 	 * IFF_ALLMULTI is requested not asking us and not reporting.
8703 	 */
8704 	if ((flags ^ dev->gflags) & IFF_ALLMULTI) {
8705 		int inc = (flags & IFF_ALLMULTI) ? 1 : -1;
8706 
8707 		dev->gflags ^= IFF_ALLMULTI;
8708 		__dev_set_allmulti(dev, inc, false);
8709 	}
8710 
8711 	return ret;
8712 }
8713 
8714 void __dev_notify_flags(struct net_device *dev, unsigned int old_flags,
8715 			unsigned int gchanges)
8716 {
8717 	unsigned int changes = dev->flags ^ old_flags;
8718 
8719 	if (gchanges)
8720 		rtmsg_ifinfo(RTM_NEWLINK, dev, gchanges, GFP_ATOMIC);
8721 
8722 	if (changes & IFF_UP) {
8723 		if (dev->flags & IFF_UP)
8724 			call_netdevice_notifiers(NETDEV_UP, dev);
8725 		else
8726 			call_netdevice_notifiers(NETDEV_DOWN, dev);
8727 	}
8728 
8729 	if (dev->flags & IFF_UP &&
8730 	    (changes & ~(IFF_UP | IFF_PROMISC | IFF_ALLMULTI | IFF_VOLATILE))) {
8731 		struct netdev_notifier_change_info change_info = {
8732 			.info = {
8733 				.dev = dev,
8734 			},
8735 			.flags_changed = changes,
8736 		};
8737 
8738 		call_netdevice_notifiers_info(NETDEV_CHANGE, &change_info.info);
8739 	}
8740 }
8741 
8742 /**
8743  *	dev_change_flags - change device settings
8744  *	@dev: device
8745  *	@flags: device state flags
8746  *	@extack: netlink extended ack
8747  *
8748  *	Change settings on device based state flags. The flags are
8749  *	in the userspace exported format.
8750  */
8751 int dev_change_flags(struct net_device *dev, unsigned int flags,
8752 		     struct netlink_ext_ack *extack)
8753 {
8754 	int ret;
8755 	unsigned int changes, old_flags = dev->flags, old_gflags = dev->gflags;
8756 
8757 	ret = __dev_change_flags(dev, flags, extack);
8758 	if (ret < 0)
8759 		return ret;
8760 
8761 	changes = (old_flags ^ dev->flags) | (old_gflags ^ dev->gflags);
8762 	__dev_notify_flags(dev, old_flags, changes);
8763 	return ret;
8764 }
8765 EXPORT_SYMBOL(dev_change_flags);
8766 
8767 int __dev_set_mtu(struct net_device *dev, int new_mtu)
8768 {
8769 	const struct net_device_ops *ops = dev->netdev_ops;
8770 
8771 	if (ops->ndo_change_mtu)
8772 		return ops->ndo_change_mtu(dev, new_mtu);
8773 
8774 	/* Pairs with all the lockless reads of dev->mtu in the stack */
8775 	WRITE_ONCE(dev->mtu, new_mtu);
8776 	return 0;
8777 }
8778 EXPORT_SYMBOL(__dev_set_mtu);
8779 
8780 int dev_validate_mtu(struct net_device *dev, int new_mtu,
8781 		     struct netlink_ext_ack *extack)
8782 {
8783 	/* MTU must be positive, and in range */
8784 	if (new_mtu < 0 || new_mtu < dev->min_mtu) {
8785 		NL_SET_ERR_MSG(extack, "mtu less than device minimum");
8786 		return -EINVAL;
8787 	}
8788 
8789 	if (dev->max_mtu > 0 && new_mtu > dev->max_mtu) {
8790 		NL_SET_ERR_MSG(extack, "mtu greater than device maximum");
8791 		return -EINVAL;
8792 	}
8793 	return 0;
8794 }
8795 
8796 /**
8797  *	dev_set_mtu_ext - Change maximum transfer unit
8798  *	@dev: device
8799  *	@new_mtu: new transfer unit
8800  *	@extack: netlink extended ack
8801  *
8802  *	Change the maximum transfer size of the network device.
8803  */
8804 int dev_set_mtu_ext(struct net_device *dev, int new_mtu,
8805 		    struct netlink_ext_ack *extack)
8806 {
8807 	int err, orig_mtu;
8808 
8809 	if (new_mtu == dev->mtu)
8810 		return 0;
8811 
8812 	err = dev_validate_mtu(dev, new_mtu, extack);
8813 	if (err)
8814 		return err;
8815 
8816 	if (!netif_device_present(dev))
8817 		return -ENODEV;
8818 
8819 	err = call_netdevice_notifiers(NETDEV_PRECHANGEMTU, dev);
8820 	err = notifier_to_errno(err);
8821 	if (err)
8822 		return err;
8823 
8824 	orig_mtu = dev->mtu;
8825 	err = __dev_set_mtu(dev, new_mtu);
8826 
8827 	if (!err) {
8828 		err = call_netdevice_notifiers_mtu(NETDEV_CHANGEMTU, dev,
8829 						   orig_mtu);
8830 		err = notifier_to_errno(err);
8831 		if (err) {
8832 			/* setting mtu back and notifying everyone again,
8833 			 * so that they have a chance to revert changes.
8834 			 */
8835 			__dev_set_mtu(dev, orig_mtu);
8836 			call_netdevice_notifiers_mtu(NETDEV_CHANGEMTU, dev,
8837 						     new_mtu);
8838 		}
8839 	}
8840 	return err;
8841 }
8842 
8843 int dev_set_mtu(struct net_device *dev, int new_mtu)
8844 {
8845 	struct netlink_ext_ack extack;
8846 	int err;
8847 
8848 	memset(&extack, 0, sizeof(extack));
8849 	err = dev_set_mtu_ext(dev, new_mtu, &extack);
8850 	if (err && extack._msg)
8851 		net_err_ratelimited("%s: %s\n", dev->name, extack._msg);
8852 	return err;
8853 }
8854 EXPORT_SYMBOL(dev_set_mtu);
8855 
8856 /**
8857  *	dev_change_tx_queue_len - Change TX queue length of a netdevice
8858  *	@dev: device
8859  *	@new_len: new tx queue length
8860  */
8861 int dev_change_tx_queue_len(struct net_device *dev, unsigned long new_len)
8862 {
8863 	unsigned int orig_len = dev->tx_queue_len;
8864 	int res;
8865 
8866 	if (new_len != (unsigned int)new_len)
8867 		return -ERANGE;
8868 
8869 	if (new_len != orig_len) {
8870 		dev->tx_queue_len = new_len;
8871 		res = call_netdevice_notifiers(NETDEV_CHANGE_TX_QUEUE_LEN, dev);
8872 		res = notifier_to_errno(res);
8873 		if (res)
8874 			goto err_rollback;
8875 		res = dev_qdisc_change_tx_queue_len(dev);
8876 		if (res)
8877 			goto err_rollback;
8878 	}
8879 
8880 	return 0;
8881 
8882 err_rollback:
8883 	netdev_err(dev, "refused to change device tx_queue_len\n");
8884 	dev->tx_queue_len = orig_len;
8885 	return res;
8886 }
8887 
8888 /**
8889  *	dev_set_group - Change group this device belongs to
8890  *	@dev: device
8891  *	@new_group: group this device should belong to
8892  */
8893 void dev_set_group(struct net_device *dev, int new_group)
8894 {
8895 	dev->group = new_group;
8896 }
8897 EXPORT_SYMBOL(dev_set_group);
8898 
8899 /**
8900  *	dev_pre_changeaddr_notify - Call NETDEV_PRE_CHANGEADDR.
8901  *	@dev: device
8902  *	@addr: new address
8903  *	@extack: netlink extended ack
8904  */
8905 int dev_pre_changeaddr_notify(struct net_device *dev, const char *addr,
8906 			      struct netlink_ext_ack *extack)
8907 {
8908 	struct netdev_notifier_pre_changeaddr_info info = {
8909 		.info.dev = dev,
8910 		.info.extack = extack,
8911 		.dev_addr = addr,
8912 	};
8913 	int rc;
8914 
8915 	rc = call_netdevice_notifiers_info(NETDEV_PRE_CHANGEADDR, &info.info);
8916 	return notifier_to_errno(rc);
8917 }
8918 EXPORT_SYMBOL(dev_pre_changeaddr_notify);
8919 
8920 /**
8921  *	dev_set_mac_address - Change Media Access Control Address
8922  *	@dev: device
8923  *	@sa: new address
8924  *	@extack: netlink extended ack
8925  *
8926  *	Change the hardware (MAC) address of the device
8927  */
8928 int dev_set_mac_address(struct net_device *dev, struct sockaddr *sa,
8929 			struct netlink_ext_ack *extack)
8930 {
8931 	const struct net_device_ops *ops = dev->netdev_ops;
8932 	int err;
8933 
8934 	if (!ops->ndo_set_mac_address)
8935 		return -EOPNOTSUPP;
8936 	if (sa->sa_family != dev->type)
8937 		return -EINVAL;
8938 	if (!netif_device_present(dev))
8939 		return -ENODEV;
8940 	err = dev_pre_changeaddr_notify(dev, sa->sa_data, extack);
8941 	if (err)
8942 		return err;
8943 	err = ops->ndo_set_mac_address(dev, sa);
8944 	if (err)
8945 		return err;
8946 	dev->addr_assign_type = NET_ADDR_SET;
8947 	call_netdevice_notifiers(NETDEV_CHANGEADDR, dev);
8948 	add_device_randomness(dev->dev_addr, dev->addr_len);
8949 	return 0;
8950 }
8951 EXPORT_SYMBOL(dev_set_mac_address);
8952 
8953 static DECLARE_RWSEM(dev_addr_sem);
8954 
8955 int dev_set_mac_address_user(struct net_device *dev, struct sockaddr *sa,
8956 			     struct netlink_ext_ack *extack)
8957 {
8958 	int ret;
8959 
8960 	down_write(&dev_addr_sem);
8961 	ret = dev_set_mac_address(dev, sa, extack);
8962 	up_write(&dev_addr_sem);
8963 	return ret;
8964 }
8965 EXPORT_SYMBOL(dev_set_mac_address_user);
8966 
8967 int dev_get_mac_address(struct sockaddr *sa, struct net *net, char *dev_name)
8968 {
8969 	size_t size = sizeof(sa->sa_data);
8970 	struct net_device *dev;
8971 	int ret = 0;
8972 
8973 	down_read(&dev_addr_sem);
8974 	rcu_read_lock();
8975 
8976 	dev = dev_get_by_name_rcu(net, dev_name);
8977 	if (!dev) {
8978 		ret = -ENODEV;
8979 		goto unlock;
8980 	}
8981 	if (!dev->addr_len)
8982 		memset(sa->sa_data, 0, size);
8983 	else
8984 		memcpy(sa->sa_data, dev->dev_addr,
8985 		       min_t(size_t, size, dev->addr_len));
8986 	sa->sa_family = dev->type;
8987 
8988 unlock:
8989 	rcu_read_unlock();
8990 	up_read(&dev_addr_sem);
8991 	return ret;
8992 }
8993 EXPORT_SYMBOL(dev_get_mac_address);
8994 
8995 /**
8996  *	dev_change_carrier - Change device carrier
8997  *	@dev: device
8998  *	@new_carrier: new value
8999  *
9000  *	Change device carrier
9001  */
9002 int dev_change_carrier(struct net_device *dev, bool new_carrier)
9003 {
9004 	const struct net_device_ops *ops = dev->netdev_ops;
9005 
9006 	if (!ops->ndo_change_carrier)
9007 		return -EOPNOTSUPP;
9008 	if (!netif_device_present(dev))
9009 		return -ENODEV;
9010 	return ops->ndo_change_carrier(dev, new_carrier);
9011 }
9012 EXPORT_SYMBOL(dev_change_carrier);
9013 
9014 /**
9015  *	dev_get_phys_port_id - Get device physical port ID
9016  *	@dev: device
9017  *	@ppid: port ID
9018  *
9019  *	Get device physical port ID
9020  */
9021 int dev_get_phys_port_id(struct net_device *dev,
9022 			 struct netdev_phys_item_id *ppid)
9023 {
9024 	const struct net_device_ops *ops = dev->netdev_ops;
9025 
9026 	if (!ops->ndo_get_phys_port_id)
9027 		return -EOPNOTSUPP;
9028 	return ops->ndo_get_phys_port_id(dev, ppid);
9029 }
9030 EXPORT_SYMBOL(dev_get_phys_port_id);
9031 
9032 /**
9033  *	dev_get_phys_port_name - Get device physical port name
9034  *	@dev: device
9035  *	@name: port name
9036  *	@len: limit of bytes to copy to name
9037  *
9038  *	Get device physical port name
9039  */
9040 int dev_get_phys_port_name(struct net_device *dev,
9041 			   char *name, size_t len)
9042 {
9043 	const struct net_device_ops *ops = dev->netdev_ops;
9044 	int err;
9045 
9046 	if (ops->ndo_get_phys_port_name) {
9047 		err = ops->ndo_get_phys_port_name(dev, name, len);
9048 		if (err != -EOPNOTSUPP)
9049 			return err;
9050 	}
9051 	return devlink_compat_phys_port_name_get(dev, name, len);
9052 }
9053 EXPORT_SYMBOL(dev_get_phys_port_name);
9054 
9055 /**
9056  *	dev_get_port_parent_id - Get the device's port parent identifier
9057  *	@dev: network device
9058  *	@ppid: pointer to a storage for the port's parent identifier
9059  *	@recurse: allow/disallow recursion to lower devices
9060  *
9061  *	Get the devices's port parent identifier
9062  */
9063 int dev_get_port_parent_id(struct net_device *dev,
9064 			   struct netdev_phys_item_id *ppid,
9065 			   bool recurse)
9066 {
9067 	const struct net_device_ops *ops = dev->netdev_ops;
9068 	struct netdev_phys_item_id first = { };
9069 	struct net_device *lower_dev;
9070 	struct list_head *iter;
9071 	int err;
9072 
9073 	if (ops->ndo_get_port_parent_id) {
9074 		err = ops->ndo_get_port_parent_id(dev, ppid);
9075 		if (err != -EOPNOTSUPP)
9076 			return err;
9077 	}
9078 
9079 	err = devlink_compat_switch_id_get(dev, ppid);
9080 	if (!err || err != -EOPNOTSUPP)
9081 		return err;
9082 
9083 	if (!recurse)
9084 		return -EOPNOTSUPP;
9085 
9086 	netdev_for_each_lower_dev(dev, lower_dev, iter) {
9087 		err = dev_get_port_parent_id(lower_dev, ppid, recurse);
9088 		if (err)
9089 			break;
9090 		if (!first.id_len)
9091 			first = *ppid;
9092 		else if (memcmp(&first, ppid, sizeof(*ppid)))
9093 			return -EOPNOTSUPP;
9094 	}
9095 
9096 	return err;
9097 }
9098 EXPORT_SYMBOL(dev_get_port_parent_id);
9099 
9100 /**
9101  *	netdev_port_same_parent_id - Indicate if two network devices have
9102  *	the same port parent identifier
9103  *	@a: first network device
9104  *	@b: second network device
9105  */
9106 bool netdev_port_same_parent_id(struct net_device *a, struct net_device *b)
9107 {
9108 	struct netdev_phys_item_id a_id = { };
9109 	struct netdev_phys_item_id b_id = { };
9110 
9111 	if (dev_get_port_parent_id(a, &a_id, true) ||
9112 	    dev_get_port_parent_id(b, &b_id, true))
9113 		return false;
9114 
9115 	return netdev_phys_item_id_same(&a_id, &b_id);
9116 }
9117 EXPORT_SYMBOL(netdev_port_same_parent_id);
9118 
9119 /**
9120  *	dev_change_proto_down - update protocol port state information
9121  *	@dev: device
9122  *	@proto_down: new value
9123  *
9124  *	This info can be used by switch drivers to set the phys state of the
9125  *	port.
9126  */
9127 int dev_change_proto_down(struct net_device *dev, bool proto_down)
9128 {
9129 	const struct net_device_ops *ops = dev->netdev_ops;
9130 
9131 	if (!ops->ndo_change_proto_down)
9132 		return -EOPNOTSUPP;
9133 	if (!netif_device_present(dev))
9134 		return -ENODEV;
9135 	return ops->ndo_change_proto_down(dev, proto_down);
9136 }
9137 EXPORT_SYMBOL(dev_change_proto_down);
9138 
9139 /**
9140  *	dev_change_proto_down_generic - generic implementation for
9141  * 	ndo_change_proto_down that sets carrier according to
9142  * 	proto_down.
9143  *
9144  *	@dev: device
9145  *	@proto_down: new value
9146  */
9147 int dev_change_proto_down_generic(struct net_device *dev, bool proto_down)
9148 {
9149 	if (proto_down)
9150 		netif_carrier_off(dev);
9151 	else
9152 		netif_carrier_on(dev);
9153 	dev->proto_down = proto_down;
9154 	return 0;
9155 }
9156 EXPORT_SYMBOL(dev_change_proto_down_generic);
9157 
9158 /**
9159  *	dev_change_proto_down_reason - proto down reason
9160  *
9161  *	@dev: device
9162  *	@mask: proto down mask
9163  *	@value: proto down value
9164  */
9165 void dev_change_proto_down_reason(struct net_device *dev, unsigned long mask,
9166 				  u32 value)
9167 {
9168 	int b;
9169 
9170 	if (!mask) {
9171 		dev->proto_down_reason = value;
9172 	} else {
9173 		for_each_set_bit(b, &mask, 32) {
9174 			if (value & (1 << b))
9175 				dev->proto_down_reason |= BIT(b);
9176 			else
9177 				dev->proto_down_reason &= ~BIT(b);
9178 		}
9179 	}
9180 }
9181 EXPORT_SYMBOL(dev_change_proto_down_reason);
9182 
9183 struct bpf_xdp_link {
9184 	struct bpf_link link;
9185 	struct net_device *dev; /* protected by rtnl_lock, no refcnt held */
9186 	int flags;
9187 };
9188 
9189 static enum bpf_xdp_mode dev_xdp_mode(struct net_device *dev, u32 flags)
9190 {
9191 	if (flags & XDP_FLAGS_HW_MODE)
9192 		return XDP_MODE_HW;
9193 	if (flags & XDP_FLAGS_DRV_MODE)
9194 		return XDP_MODE_DRV;
9195 	if (flags & XDP_FLAGS_SKB_MODE)
9196 		return XDP_MODE_SKB;
9197 	return dev->netdev_ops->ndo_bpf ? XDP_MODE_DRV : XDP_MODE_SKB;
9198 }
9199 
9200 static bpf_op_t dev_xdp_bpf_op(struct net_device *dev, enum bpf_xdp_mode mode)
9201 {
9202 	switch (mode) {
9203 	case XDP_MODE_SKB:
9204 		return generic_xdp_install;
9205 	case XDP_MODE_DRV:
9206 	case XDP_MODE_HW:
9207 		return dev->netdev_ops->ndo_bpf;
9208 	default:
9209 		return NULL;
9210 	}
9211 }
9212 
9213 static struct bpf_xdp_link *dev_xdp_link(struct net_device *dev,
9214 					 enum bpf_xdp_mode mode)
9215 {
9216 	return dev->xdp_state[mode].link;
9217 }
9218 
9219 static struct bpf_prog *dev_xdp_prog(struct net_device *dev,
9220 				     enum bpf_xdp_mode mode)
9221 {
9222 	struct bpf_xdp_link *link = dev_xdp_link(dev, mode);
9223 
9224 	if (link)
9225 		return link->link.prog;
9226 	return dev->xdp_state[mode].prog;
9227 }
9228 
9229 static u8 dev_xdp_prog_count(struct net_device *dev)
9230 {
9231 	u8 count = 0;
9232 	int i;
9233 
9234 	for (i = 0; i < __MAX_XDP_MODE; i++)
9235 		if (dev->xdp_state[i].prog || dev->xdp_state[i].link)
9236 			count++;
9237 	return count;
9238 }
9239 
9240 u32 dev_xdp_prog_id(struct net_device *dev, enum bpf_xdp_mode mode)
9241 {
9242 	struct bpf_prog *prog = dev_xdp_prog(dev, mode);
9243 
9244 	return prog ? prog->aux->id : 0;
9245 }
9246 
9247 static void dev_xdp_set_link(struct net_device *dev, enum bpf_xdp_mode mode,
9248 			     struct bpf_xdp_link *link)
9249 {
9250 	dev->xdp_state[mode].link = link;
9251 	dev->xdp_state[mode].prog = NULL;
9252 }
9253 
9254 static void dev_xdp_set_prog(struct net_device *dev, enum bpf_xdp_mode mode,
9255 			     struct bpf_prog *prog)
9256 {
9257 	dev->xdp_state[mode].link = NULL;
9258 	dev->xdp_state[mode].prog = prog;
9259 }
9260 
9261 static int dev_xdp_install(struct net_device *dev, enum bpf_xdp_mode mode,
9262 			   bpf_op_t bpf_op, struct netlink_ext_ack *extack,
9263 			   u32 flags, struct bpf_prog *prog)
9264 {
9265 	struct netdev_bpf xdp;
9266 	int err;
9267 
9268 	memset(&xdp, 0, sizeof(xdp));
9269 	xdp.command = mode == XDP_MODE_HW ? XDP_SETUP_PROG_HW : XDP_SETUP_PROG;
9270 	xdp.extack = extack;
9271 	xdp.flags = flags;
9272 	xdp.prog = prog;
9273 
9274 	/* Drivers assume refcnt is already incremented (i.e, prog pointer is
9275 	 * "moved" into driver), so they don't increment it on their own, but
9276 	 * they do decrement refcnt when program is detached or replaced.
9277 	 * Given net_device also owns link/prog, we need to bump refcnt here
9278 	 * to prevent drivers from underflowing it.
9279 	 */
9280 	if (prog)
9281 		bpf_prog_inc(prog);
9282 	err = bpf_op(dev, &xdp);
9283 	if (err) {
9284 		if (prog)
9285 			bpf_prog_put(prog);
9286 		return err;
9287 	}
9288 
9289 	if (mode != XDP_MODE_HW)
9290 		bpf_prog_change_xdp(dev_xdp_prog(dev, mode), prog);
9291 
9292 	return 0;
9293 }
9294 
9295 static void dev_xdp_uninstall(struct net_device *dev)
9296 {
9297 	struct bpf_xdp_link *link;
9298 	struct bpf_prog *prog;
9299 	enum bpf_xdp_mode mode;
9300 	bpf_op_t bpf_op;
9301 
9302 	ASSERT_RTNL();
9303 
9304 	for (mode = XDP_MODE_SKB; mode < __MAX_XDP_MODE; mode++) {
9305 		prog = dev_xdp_prog(dev, mode);
9306 		if (!prog)
9307 			continue;
9308 
9309 		bpf_op = dev_xdp_bpf_op(dev, mode);
9310 		if (!bpf_op)
9311 			continue;
9312 
9313 		WARN_ON(dev_xdp_install(dev, mode, bpf_op, NULL, 0, NULL));
9314 
9315 		/* auto-detach link from net device */
9316 		link = dev_xdp_link(dev, mode);
9317 		if (link)
9318 			link->dev = NULL;
9319 		else
9320 			bpf_prog_put(prog);
9321 
9322 		dev_xdp_set_link(dev, mode, NULL);
9323 	}
9324 }
9325 
9326 static int dev_xdp_attach(struct net_device *dev, struct netlink_ext_ack *extack,
9327 			  struct bpf_xdp_link *link, struct bpf_prog *new_prog,
9328 			  struct bpf_prog *old_prog, u32 flags)
9329 {
9330 	unsigned int num_modes = hweight32(flags & XDP_FLAGS_MODES);
9331 	struct bpf_prog *cur_prog;
9332 	enum bpf_xdp_mode mode;
9333 	bpf_op_t bpf_op;
9334 	int err;
9335 
9336 	ASSERT_RTNL();
9337 
9338 	/* either link or prog attachment, never both */
9339 	if (link && (new_prog || old_prog))
9340 		return -EINVAL;
9341 	/* link supports only XDP mode flags */
9342 	if (link && (flags & ~XDP_FLAGS_MODES)) {
9343 		NL_SET_ERR_MSG(extack, "Invalid XDP flags for BPF link attachment");
9344 		return -EINVAL;
9345 	}
9346 	/* just one XDP mode bit should be set, zero defaults to drv/skb mode */
9347 	if (num_modes > 1) {
9348 		NL_SET_ERR_MSG(extack, "Only one XDP mode flag can be set");
9349 		return -EINVAL;
9350 	}
9351 	/* avoid ambiguity if offload + drv/skb mode progs are both loaded */
9352 	if (!num_modes && dev_xdp_prog_count(dev) > 1) {
9353 		NL_SET_ERR_MSG(extack,
9354 			       "More than one program loaded, unset mode is ambiguous");
9355 		return -EINVAL;
9356 	}
9357 	/* old_prog != NULL implies XDP_FLAGS_REPLACE is set */
9358 	if (old_prog && !(flags & XDP_FLAGS_REPLACE)) {
9359 		NL_SET_ERR_MSG(extack, "XDP_FLAGS_REPLACE is not specified");
9360 		return -EINVAL;
9361 	}
9362 
9363 	mode = dev_xdp_mode(dev, flags);
9364 	/* can't replace attached link */
9365 	if (dev_xdp_link(dev, mode)) {
9366 		NL_SET_ERR_MSG(extack, "Can't replace active BPF XDP link");
9367 		return -EBUSY;
9368 	}
9369 
9370 	cur_prog = dev_xdp_prog(dev, mode);
9371 	/* can't replace attached prog with link */
9372 	if (link && cur_prog) {
9373 		NL_SET_ERR_MSG(extack, "Can't replace active XDP program with BPF link");
9374 		return -EBUSY;
9375 	}
9376 	if ((flags & XDP_FLAGS_REPLACE) && cur_prog != old_prog) {
9377 		NL_SET_ERR_MSG(extack, "Active program does not match expected");
9378 		return -EEXIST;
9379 	}
9380 
9381 	/* put effective new program into new_prog */
9382 	if (link)
9383 		new_prog = link->link.prog;
9384 
9385 	if (new_prog) {
9386 		bool offload = mode == XDP_MODE_HW;
9387 		enum bpf_xdp_mode other_mode = mode == XDP_MODE_SKB
9388 					       ? XDP_MODE_DRV : XDP_MODE_SKB;
9389 
9390 		if ((flags & XDP_FLAGS_UPDATE_IF_NOEXIST) && cur_prog) {
9391 			NL_SET_ERR_MSG(extack, "XDP program already attached");
9392 			return -EBUSY;
9393 		}
9394 		if (!offload && dev_xdp_prog(dev, other_mode)) {
9395 			NL_SET_ERR_MSG(extack, "Native and generic XDP can't be active at the same time");
9396 			return -EEXIST;
9397 		}
9398 		if (!offload && bpf_prog_is_dev_bound(new_prog->aux)) {
9399 			NL_SET_ERR_MSG(extack, "Using device-bound program without HW_MODE flag is not supported");
9400 			return -EINVAL;
9401 		}
9402 		if (new_prog->expected_attach_type == BPF_XDP_DEVMAP) {
9403 			NL_SET_ERR_MSG(extack, "BPF_XDP_DEVMAP programs can not be attached to a device");
9404 			return -EINVAL;
9405 		}
9406 		if (new_prog->expected_attach_type == BPF_XDP_CPUMAP) {
9407 			NL_SET_ERR_MSG(extack, "BPF_XDP_CPUMAP programs can not be attached to a device");
9408 			return -EINVAL;
9409 		}
9410 	}
9411 
9412 	/* don't call drivers if the effective program didn't change */
9413 	if (new_prog != cur_prog) {
9414 		bpf_op = dev_xdp_bpf_op(dev, mode);
9415 		if (!bpf_op) {
9416 			NL_SET_ERR_MSG(extack, "Underlying driver does not support XDP in native mode");
9417 			return -EOPNOTSUPP;
9418 		}
9419 
9420 		err = dev_xdp_install(dev, mode, bpf_op, extack, flags, new_prog);
9421 		if (err)
9422 			return err;
9423 	}
9424 
9425 	if (link)
9426 		dev_xdp_set_link(dev, mode, link);
9427 	else
9428 		dev_xdp_set_prog(dev, mode, new_prog);
9429 	if (cur_prog)
9430 		bpf_prog_put(cur_prog);
9431 
9432 	return 0;
9433 }
9434 
9435 static int dev_xdp_attach_link(struct net_device *dev,
9436 			       struct netlink_ext_ack *extack,
9437 			       struct bpf_xdp_link *link)
9438 {
9439 	return dev_xdp_attach(dev, extack, link, NULL, NULL, link->flags);
9440 }
9441 
9442 static int dev_xdp_detach_link(struct net_device *dev,
9443 			       struct netlink_ext_ack *extack,
9444 			       struct bpf_xdp_link *link)
9445 {
9446 	enum bpf_xdp_mode mode;
9447 	bpf_op_t bpf_op;
9448 
9449 	ASSERT_RTNL();
9450 
9451 	mode = dev_xdp_mode(dev, link->flags);
9452 	if (dev_xdp_link(dev, mode) != link)
9453 		return -EINVAL;
9454 
9455 	bpf_op = dev_xdp_bpf_op(dev, mode);
9456 	WARN_ON(dev_xdp_install(dev, mode, bpf_op, NULL, 0, NULL));
9457 	dev_xdp_set_link(dev, mode, NULL);
9458 	return 0;
9459 }
9460 
9461 static void bpf_xdp_link_release(struct bpf_link *link)
9462 {
9463 	struct bpf_xdp_link *xdp_link = container_of(link, struct bpf_xdp_link, link);
9464 
9465 	rtnl_lock();
9466 
9467 	/* if racing with net_device's tear down, xdp_link->dev might be
9468 	 * already NULL, in which case link was already auto-detached
9469 	 */
9470 	if (xdp_link->dev) {
9471 		WARN_ON(dev_xdp_detach_link(xdp_link->dev, NULL, xdp_link));
9472 		xdp_link->dev = NULL;
9473 	}
9474 
9475 	rtnl_unlock();
9476 }
9477 
9478 static int bpf_xdp_link_detach(struct bpf_link *link)
9479 {
9480 	bpf_xdp_link_release(link);
9481 	return 0;
9482 }
9483 
9484 static void bpf_xdp_link_dealloc(struct bpf_link *link)
9485 {
9486 	struct bpf_xdp_link *xdp_link = container_of(link, struct bpf_xdp_link, link);
9487 
9488 	kfree(xdp_link);
9489 }
9490 
9491 static void bpf_xdp_link_show_fdinfo(const struct bpf_link *link,
9492 				     struct seq_file *seq)
9493 {
9494 	struct bpf_xdp_link *xdp_link = container_of(link, struct bpf_xdp_link, link);
9495 	u32 ifindex = 0;
9496 
9497 	rtnl_lock();
9498 	if (xdp_link->dev)
9499 		ifindex = xdp_link->dev->ifindex;
9500 	rtnl_unlock();
9501 
9502 	seq_printf(seq, "ifindex:\t%u\n", ifindex);
9503 }
9504 
9505 static int bpf_xdp_link_fill_link_info(const struct bpf_link *link,
9506 				       struct bpf_link_info *info)
9507 {
9508 	struct bpf_xdp_link *xdp_link = container_of(link, struct bpf_xdp_link, link);
9509 	u32 ifindex = 0;
9510 
9511 	rtnl_lock();
9512 	if (xdp_link->dev)
9513 		ifindex = xdp_link->dev->ifindex;
9514 	rtnl_unlock();
9515 
9516 	info->xdp.ifindex = ifindex;
9517 	return 0;
9518 }
9519 
9520 static int bpf_xdp_link_update(struct bpf_link *link, struct bpf_prog *new_prog,
9521 			       struct bpf_prog *old_prog)
9522 {
9523 	struct bpf_xdp_link *xdp_link = container_of(link, struct bpf_xdp_link, link);
9524 	enum bpf_xdp_mode mode;
9525 	bpf_op_t bpf_op;
9526 	int err = 0;
9527 
9528 	rtnl_lock();
9529 
9530 	/* link might have been auto-released already, so fail */
9531 	if (!xdp_link->dev) {
9532 		err = -ENOLINK;
9533 		goto out_unlock;
9534 	}
9535 
9536 	if (old_prog && link->prog != old_prog) {
9537 		err = -EPERM;
9538 		goto out_unlock;
9539 	}
9540 	old_prog = link->prog;
9541 	if (old_prog == new_prog) {
9542 		/* no-op, don't disturb drivers */
9543 		bpf_prog_put(new_prog);
9544 		goto out_unlock;
9545 	}
9546 
9547 	mode = dev_xdp_mode(xdp_link->dev, xdp_link->flags);
9548 	bpf_op = dev_xdp_bpf_op(xdp_link->dev, mode);
9549 	err = dev_xdp_install(xdp_link->dev, mode, bpf_op, NULL,
9550 			      xdp_link->flags, new_prog);
9551 	if (err)
9552 		goto out_unlock;
9553 
9554 	old_prog = xchg(&link->prog, new_prog);
9555 	bpf_prog_put(old_prog);
9556 
9557 out_unlock:
9558 	rtnl_unlock();
9559 	return err;
9560 }
9561 
9562 static const struct bpf_link_ops bpf_xdp_link_lops = {
9563 	.release = bpf_xdp_link_release,
9564 	.dealloc = bpf_xdp_link_dealloc,
9565 	.detach = bpf_xdp_link_detach,
9566 	.show_fdinfo = bpf_xdp_link_show_fdinfo,
9567 	.fill_link_info = bpf_xdp_link_fill_link_info,
9568 	.update_prog = bpf_xdp_link_update,
9569 };
9570 
9571 int bpf_xdp_link_attach(const union bpf_attr *attr, struct bpf_prog *prog)
9572 {
9573 	struct net *net = current->nsproxy->net_ns;
9574 	struct bpf_link_primer link_primer;
9575 	struct bpf_xdp_link *link;
9576 	struct net_device *dev;
9577 	int err, fd;
9578 
9579 	dev = dev_get_by_index(net, attr->link_create.target_ifindex);
9580 	if (!dev)
9581 		return -EINVAL;
9582 
9583 	link = kzalloc(sizeof(*link), GFP_USER);
9584 	if (!link) {
9585 		err = -ENOMEM;
9586 		goto out_put_dev;
9587 	}
9588 
9589 	bpf_link_init(&link->link, BPF_LINK_TYPE_XDP, &bpf_xdp_link_lops, prog);
9590 	link->dev = dev;
9591 	link->flags = attr->link_create.flags;
9592 
9593 	err = bpf_link_prime(&link->link, &link_primer);
9594 	if (err) {
9595 		kfree(link);
9596 		goto out_put_dev;
9597 	}
9598 
9599 	rtnl_lock();
9600 	err = dev_xdp_attach_link(dev, NULL, link);
9601 	rtnl_unlock();
9602 
9603 	if (err) {
9604 		bpf_link_cleanup(&link_primer);
9605 		goto out_put_dev;
9606 	}
9607 
9608 	fd = bpf_link_settle(&link_primer);
9609 	/* link itself doesn't hold dev's refcnt to not complicate shutdown */
9610 	dev_put(dev);
9611 	return fd;
9612 
9613 out_put_dev:
9614 	dev_put(dev);
9615 	return err;
9616 }
9617 
9618 /**
9619  *	dev_change_xdp_fd - set or clear a bpf program for a device rx path
9620  *	@dev: device
9621  *	@extack: netlink extended ack
9622  *	@fd: new program fd or negative value to clear
9623  *	@expected_fd: old program fd that userspace expects to replace or clear
9624  *	@flags: xdp-related flags
9625  *
9626  *	Set or clear a bpf program for a device
9627  */
9628 int dev_change_xdp_fd(struct net_device *dev, struct netlink_ext_ack *extack,
9629 		      int fd, int expected_fd, u32 flags)
9630 {
9631 	enum bpf_xdp_mode mode = dev_xdp_mode(dev, flags);
9632 	struct bpf_prog *new_prog = NULL, *old_prog = NULL;
9633 	int err;
9634 
9635 	ASSERT_RTNL();
9636 
9637 	if (fd >= 0) {
9638 		new_prog = bpf_prog_get_type_dev(fd, BPF_PROG_TYPE_XDP,
9639 						 mode != XDP_MODE_SKB);
9640 		if (IS_ERR(new_prog))
9641 			return PTR_ERR(new_prog);
9642 	}
9643 
9644 	if (expected_fd >= 0) {
9645 		old_prog = bpf_prog_get_type_dev(expected_fd, BPF_PROG_TYPE_XDP,
9646 						 mode != XDP_MODE_SKB);
9647 		if (IS_ERR(old_prog)) {
9648 			err = PTR_ERR(old_prog);
9649 			old_prog = NULL;
9650 			goto err_out;
9651 		}
9652 	}
9653 
9654 	err = dev_xdp_attach(dev, extack, NULL, new_prog, old_prog, flags);
9655 
9656 err_out:
9657 	if (err && new_prog)
9658 		bpf_prog_put(new_prog);
9659 	if (old_prog)
9660 		bpf_prog_put(old_prog);
9661 	return err;
9662 }
9663 
9664 /**
9665  *	dev_new_index	-	allocate an ifindex
9666  *	@net: the applicable net namespace
9667  *
9668  *	Returns a suitable unique value for a new device interface
9669  *	number.  The caller must hold the rtnl semaphore or the
9670  *	dev_base_lock to be sure it remains unique.
9671  */
9672 static int dev_new_index(struct net *net)
9673 {
9674 	int ifindex = net->ifindex;
9675 
9676 	for (;;) {
9677 		if (++ifindex <= 0)
9678 			ifindex = 1;
9679 		if (!__dev_get_by_index(net, ifindex))
9680 			return net->ifindex = ifindex;
9681 	}
9682 }
9683 
9684 /* Delayed registration/unregisteration */
9685 static LIST_HEAD(net_todo_list);
9686 DECLARE_WAIT_QUEUE_HEAD(netdev_unregistering_wq);
9687 
9688 static void net_set_todo(struct net_device *dev)
9689 {
9690 	list_add_tail(&dev->todo_list, &net_todo_list);
9691 	dev_net(dev)->dev_unreg_count++;
9692 }
9693 
9694 static netdev_features_t netdev_sync_upper_features(struct net_device *lower,
9695 	struct net_device *upper, netdev_features_t features)
9696 {
9697 	netdev_features_t upper_disables = NETIF_F_UPPER_DISABLES;
9698 	netdev_features_t feature;
9699 	int feature_bit;
9700 
9701 	for_each_netdev_feature(upper_disables, feature_bit) {
9702 		feature = __NETIF_F_BIT(feature_bit);
9703 		if (!(upper->wanted_features & feature)
9704 		    && (features & feature)) {
9705 			netdev_dbg(lower, "Dropping feature %pNF, upper dev %s has it off.\n",
9706 				   &feature, upper->name);
9707 			features &= ~feature;
9708 		}
9709 	}
9710 
9711 	return features;
9712 }
9713 
9714 static void netdev_sync_lower_features(struct net_device *upper,
9715 	struct net_device *lower, netdev_features_t features)
9716 {
9717 	netdev_features_t upper_disables = NETIF_F_UPPER_DISABLES;
9718 	netdev_features_t feature;
9719 	int feature_bit;
9720 
9721 	for_each_netdev_feature(upper_disables, feature_bit) {
9722 		feature = __NETIF_F_BIT(feature_bit);
9723 		if (!(features & feature) && (lower->features & feature)) {
9724 			netdev_dbg(upper, "Disabling feature %pNF on lower dev %s.\n",
9725 				   &feature, lower->name);
9726 			lower->wanted_features &= ~feature;
9727 			__netdev_update_features(lower);
9728 
9729 			if (unlikely(lower->features & feature))
9730 				netdev_WARN(upper, "failed to disable %pNF on %s!\n",
9731 					    &feature, lower->name);
9732 			else
9733 				netdev_features_change(lower);
9734 		}
9735 	}
9736 }
9737 
9738 static netdev_features_t netdev_fix_features(struct net_device *dev,
9739 	netdev_features_t features)
9740 {
9741 	/* Fix illegal checksum combinations */
9742 	if ((features & NETIF_F_HW_CSUM) &&
9743 	    (features & (NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM))) {
9744 		netdev_warn(dev, "mixed HW and IP checksum settings.\n");
9745 		features &= ~(NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM);
9746 	}
9747 
9748 	/* TSO requires that SG is present as well. */
9749 	if ((features & NETIF_F_ALL_TSO) && !(features & NETIF_F_SG)) {
9750 		netdev_dbg(dev, "Dropping TSO features since no SG feature.\n");
9751 		features &= ~NETIF_F_ALL_TSO;
9752 	}
9753 
9754 	if ((features & NETIF_F_TSO) && !(features & NETIF_F_HW_CSUM) &&
9755 					!(features & NETIF_F_IP_CSUM)) {
9756 		netdev_dbg(dev, "Dropping TSO features since no CSUM feature.\n");
9757 		features &= ~NETIF_F_TSO;
9758 		features &= ~NETIF_F_TSO_ECN;
9759 	}
9760 
9761 	if ((features & NETIF_F_TSO6) && !(features & NETIF_F_HW_CSUM) &&
9762 					 !(features & NETIF_F_IPV6_CSUM)) {
9763 		netdev_dbg(dev, "Dropping TSO6 features since no CSUM feature.\n");
9764 		features &= ~NETIF_F_TSO6;
9765 	}
9766 
9767 	/* TSO with IPv4 ID mangling requires IPv4 TSO be enabled */
9768 	if ((features & NETIF_F_TSO_MANGLEID) && !(features & NETIF_F_TSO))
9769 		features &= ~NETIF_F_TSO_MANGLEID;
9770 
9771 	/* TSO ECN requires that TSO is present as well. */
9772 	if ((features & NETIF_F_ALL_TSO) == NETIF_F_TSO_ECN)
9773 		features &= ~NETIF_F_TSO_ECN;
9774 
9775 	/* Software GSO depends on SG. */
9776 	if ((features & NETIF_F_GSO) && !(features & NETIF_F_SG)) {
9777 		netdev_dbg(dev, "Dropping NETIF_F_GSO since no SG feature.\n");
9778 		features &= ~NETIF_F_GSO;
9779 	}
9780 
9781 	/* GSO partial features require GSO partial be set */
9782 	if ((features & dev->gso_partial_features) &&
9783 	    !(features & NETIF_F_GSO_PARTIAL)) {
9784 		netdev_dbg(dev,
9785 			   "Dropping partially supported GSO features since no GSO partial.\n");
9786 		features &= ~dev->gso_partial_features;
9787 	}
9788 
9789 	if (!(features & NETIF_F_RXCSUM)) {
9790 		/* NETIF_F_GRO_HW implies doing RXCSUM since every packet
9791 		 * successfully merged by hardware must also have the
9792 		 * checksum verified by hardware.  If the user does not
9793 		 * want to enable RXCSUM, logically, we should disable GRO_HW.
9794 		 */
9795 		if (features & NETIF_F_GRO_HW) {
9796 			netdev_dbg(dev, "Dropping NETIF_F_GRO_HW since no RXCSUM feature.\n");
9797 			features &= ~NETIF_F_GRO_HW;
9798 		}
9799 	}
9800 
9801 	/* LRO/HW-GRO features cannot be combined with RX-FCS */
9802 	if (features & NETIF_F_RXFCS) {
9803 		if (features & NETIF_F_LRO) {
9804 			netdev_dbg(dev, "Dropping LRO feature since RX-FCS is requested.\n");
9805 			features &= ~NETIF_F_LRO;
9806 		}
9807 
9808 		if (features & NETIF_F_GRO_HW) {
9809 			netdev_dbg(dev, "Dropping HW-GRO feature since RX-FCS is requested.\n");
9810 			features &= ~NETIF_F_GRO_HW;
9811 		}
9812 	}
9813 
9814 	if (features & NETIF_F_HW_TLS_TX) {
9815 		bool ip_csum = (features & (NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM)) ==
9816 			(NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM);
9817 		bool hw_csum = features & NETIF_F_HW_CSUM;
9818 
9819 		if (!ip_csum && !hw_csum) {
9820 			netdev_dbg(dev, "Dropping TLS TX HW offload feature since no CSUM feature.\n");
9821 			features &= ~NETIF_F_HW_TLS_TX;
9822 		}
9823 	}
9824 
9825 	if ((features & NETIF_F_HW_TLS_RX) && !(features & NETIF_F_RXCSUM)) {
9826 		netdev_dbg(dev, "Dropping TLS RX HW offload feature since no RXCSUM feature.\n");
9827 		features &= ~NETIF_F_HW_TLS_RX;
9828 	}
9829 
9830 	return features;
9831 }
9832 
9833 int __netdev_update_features(struct net_device *dev)
9834 {
9835 	struct net_device *upper, *lower;
9836 	netdev_features_t features;
9837 	struct list_head *iter;
9838 	int err = -1;
9839 
9840 	ASSERT_RTNL();
9841 
9842 	features = netdev_get_wanted_features(dev);
9843 
9844 	if (dev->netdev_ops->ndo_fix_features)
9845 		features = dev->netdev_ops->ndo_fix_features(dev, features);
9846 
9847 	/* driver might be less strict about feature dependencies */
9848 	features = netdev_fix_features(dev, features);
9849 
9850 	/* some features can't be enabled if they're off on an upper device */
9851 	netdev_for_each_upper_dev_rcu(dev, upper, iter)
9852 		features = netdev_sync_upper_features(dev, upper, features);
9853 
9854 	if (dev->features == features)
9855 		goto sync_lower;
9856 
9857 	netdev_dbg(dev, "Features changed: %pNF -> %pNF\n",
9858 		&dev->features, &features);
9859 
9860 	if (dev->netdev_ops->ndo_set_features)
9861 		err = dev->netdev_ops->ndo_set_features(dev, features);
9862 	else
9863 		err = 0;
9864 
9865 	if (unlikely(err < 0)) {
9866 		netdev_err(dev,
9867 			"set_features() failed (%d); wanted %pNF, left %pNF\n",
9868 			err, &features, &dev->features);
9869 		/* return non-0 since some features might have changed and
9870 		 * it's better to fire a spurious notification than miss it
9871 		 */
9872 		return -1;
9873 	}
9874 
9875 sync_lower:
9876 	/* some features must be disabled on lower devices when disabled
9877 	 * on an upper device (think: bonding master or bridge)
9878 	 */
9879 	netdev_for_each_lower_dev(dev, lower, iter)
9880 		netdev_sync_lower_features(dev, lower, features);
9881 
9882 	if (!err) {
9883 		netdev_features_t diff = features ^ dev->features;
9884 
9885 		if (diff & NETIF_F_RX_UDP_TUNNEL_PORT) {
9886 			/* udp_tunnel_{get,drop}_rx_info both need
9887 			 * NETIF_F_RX_UDP_TUNNEL_PORT enabled on the
9888 			 * device, or they won't do anything.
9889 			 * Thus we need to update dev->features
9890 			 * *before* calling udp_tunnel_get_rx_info,
9891 			 * but *after* calling udp_tunnel_drop_rx_info.
9892 			 */
9893 			if (features & NETIF_F_RX_UDP_TUNNEL_PORT) {
9894 				dev->features = features;
9895 				udp_tunnel_get_rx_info(dev);
9896 			} else {
9897 				udp_tunnel_drop_rx_info(dev);
9898 			}
9899 		}
9900 
9901 		if (diff & NETIF_F_HW_VLAN_CTAG_FILTER) {
9902 			if (features & NETIF_F_HW_VLAN_CTAG_FILTER) {
9903 				dev->features = features;
9904 				err |= vlan_get_rx_ctag_filter_info(dev);
9905 			} else {
9906 				vlan_drop_rx_ctag_filter_info(dev);
9907 			}
9908 		}
9909 
9910 		if (diff & NETIF_F_HW_VLAN_STAG_FILTER) {
9911 			if (features & NETIF_F_HW_VLAN_STAG_FILTER) {
9912 				dev->features = features;
9913 				err |= vlan_get_rx_stag_filter_info(dev);
9914 			} else {
9915 				vlan_drop_rx_stag_filter_info(dev);
9916 			}
9917 		}
9918 
9919 		dev->features = features;
9920 	}
9921 
9922 	return err < 0 ? 0 : 1;
9923 }
9924 
9925 /**
9926  *	netdev_update_features - recalculate device features
9927  *	@dev: the device to check
9928  *
9929  *	Recalculate dev->features set and send notifications if it
9930  *	has changed. Should be called after driver or hardware dependent
9931  *	conditions might have changed that influence the features.
9932  */
9933 void netdev_update_features(struct net_device *dev)
9934 {
9935 	if (__netdev_update_features(dev))
9936 		netdev_features_change(dev);
9937 }
9938 EXPORT_SYMBOL(netdev_update_features);
9939 
9940 /**
9941  *	netdev_change_features - recalculate device features
9942  *	@dev: the device to check
9943  *
9944  *	Recalculate dev->features set and send notifications even
9945  *	if they have not changed. Should be called instead of
9946  *	netdev_update_features() if also dev->vlan_features might
9947  *	have changed to allow the changes to be propagated to stacked
9948  *	VLAN devices.
9949  */
9950 void netdev_change_features(struct net_device *dev)
9951 {
9952 	__netdev_update_features(dev);
9953 	netdev_features_change(dev);
9954 }
9955 EXPORT_SYMBOL(netdev_change_features);
9956 
9957 /**
9958  *	netif_stacked_transfer_operstate -	transfer operstate
9959  *	@rootdev: the root or lower level device to transfer state from
9960  *	@dev: the device to transfer operstate to
9961  *
9962  *	Transfer operational state from root to device. This is normally
9963  *	called when a stacking relationship exists between the root
9964  *	device and the device(a leaf device).
9965  */
9966 void netif_stacked_transfer_operstate(const struct net_device *rootdev,
9967 					struct net_device *dev)
9968 {
9969 	if (rootdev->operstate == IF_OPER_DORMANT)
9970 		netif_dormant_on(dev);
9971 	else
9972 		netif_dormant_off(dev);
9973 
9974 	if (rootdev->operstate == IF_OPER_TESTING)
9975 		netif_testing_on(dev);
9976 	else
9977 		netif_testing_off(dev);
9978 
9979 	if (netif_carrier_ok(rootdev))
9980 		netif_carrier_on(dev);
9981 	else
9982 		netif_carrier_off(dev);
9983 }
9984 EXPORT_SYMBOL(netif_stacked_transfer_operstate);
9985 
9986 static int netif_alloc_rx_queues(struct net_device *dev)
9987 {
9988 	unsigned int i, count = dev->num_rx_queues;
9989 	struct netdev_rx_queue *rx;
9990 	size_t sz = count * sizeof(*rx);
9991 	int err = 0;
9992 
9993 	BUG_ON(count < 1);
9994 
9995 	rx = kvzalloc(sz, GFP_KERNEL | __GFP_RETRY_MAYFAIL);
9996 	if (!rx)
9997 		return -ENOMEM;
9998 
9999 	dev->_rx = rx;
10000 
10001 	for (i = 0; i < count; i++) {
10002 		rx[i].dev = dev;
10003 
10004 		/* XDP RX-queue setup */
10005 		err = xdp_rxq_info_reg(&rx[i].xdp_rxq, dev, i, 0);
10006 		if (err < 0)
10007 			goto err_rxq_info;
10008 	}
10009 	return 0;
10010 
10011 err_rxq_info:
10012 	/* Rollback successful reg's and free other resources */
10013 	while (i--)
10014 		xdp_rxq_info_unreg(&rx[i].xdp_rxq);
10015 	kvfree(dev->_rx);
10016 	dev->_rx = NULL;
10017 	return err;
10018 }
10019 
10020 static void netif_free_rx_queues(struct net_device *dev)
10021 {
10022 	unsigned int i, count = dev->num_rx_queues;
10023 
10024 	/* netif_alloc_rx_queues alloc failed, resources have been unreg'ed */
10025 	if (!dev->_rx)
10026 		return;
10027 
10028 	for (i = 0; i < count; i++)
10029 		xdp_rxq_info_unreg(&dev->_rx[i].xdp_rxq);
10030 
10031 	kvfree(dev->_rx);
10032 }
10033 
10034 static void netdev_init_one_queue(struct net_device *dev,
10035 				  struct netdev_queue *queue, void *_unused)
10036 {
10037 	/* Initialize queue lock */
10038 	spin_lock_init(&queue->_xmit_lock);
10039 	netdev_set_xmit_lockdep_class(&queue->_xmit_lock, dev->type);
10040 	queue->xmit_lock_owner = -1;
10041 	netdev_queue_numa_node_write(queue, NUMA_NO_NODE);
10042 	queue->dev = dev;
10043 #ifdef CONFIG_BQL
10044 	dql_init(&queue->dql, HZ);
10045 #endif
10046 }
10047 
10048 static void netif_free_tx_queues(struct net_device *dev)
10049 {
10050 	kvfree(dev->_tx);
10051 }
10052 
10053 static int netif_alloc_netdev_queues(struct net_device *dev)
10054 {
10055 	unsigned int count = dev->num_tx_queues;
10056 	struct netdev_queue *tx;
10057 	size_t sz = count * sizeof(*tx);
10058 
10059 	if (count < 1 || count > 0xffff)
10060 		return -EINVAL;
10061 
10062 	tx = kvzalloc(sz, GFP_KERNEL | __GFP_RETRY_MAYFAIL);
10063 	if (!tx)
10064 		return -ENOMEM;
10065 
10066 	dev->_tx = tx;
10067 
10068 	netdev_for_each_tx_queue(dev, netdev_init_one_queue, NULL);
10069 	spin_lock_init(&dev->tx_global_lock);
10070 
10071 	return 0;
10072 }
10073 
10074 void netif_tx_stop_all_queues(struct net_device *dev)
10075 {
10076 	unsigned int i;
10077 
10078 	for (i = 0; i < dev->num_tx_queues; i++) {
10079 		struct netdev_queue *txq = netdev_get_tx_queue(dev, i);
10080 
10081 		netif_tx_stop_queue(txq);
10082 	}
10083 }
10084 EXPORT_SYMBOL(netif_tx_stop_all_queues);
10085 
10086 /**
10087  *	register_netdevice	- register a network device
10088  *	@dev: device to register
10089  *
10090  *	Take a completed network device structure and add it to the kernel
10091  *	interfaces. A %NETDEV_REGISTER message is sent to the netdev notifier
10092  *	chain. 0 is returned on success. A negative errno code is returned
10093  *	on a failure to set up the device, or if the name is a duplicate.
10094  *
10095  *	Callers must hold the rtnl semaphore. You may want
10096  *	register_netdev() instead of this.
10097  *
10098  *	BUGS:
10099  *	The locking appears insufficient to guarantee two parallel registers
10100  *	will not get the same name.
10101  */
10102 
10103 int register_netdevice(struct net_device *dev)
10104 {
10105 	int ret;
10106 	struct net *net = dev_net(dev);
10107 
10108 	BUILD_BUG_ON(sizeof(netdev_features_t) * BITS_PER_BYTE <
10109 		     NETDEV_FEATURE_COUNT);
10110 	BUG_ON(dev_boot_phase);
10111 	ASSERT_RTNL();
10112 
10113 	might_sleep();
10114 
10115 	/* When net_device's are persistent, this will be fatal. */
10116 	BUG_ON(dev->reg_state != NETREG_UNINITIALIZED);
10117 	BUG_ON(!net);
10118 
10119 	ret = ethtool_check_ops(dev->ethtool_ops);
10120 	if (ret)
10121 		return ret;
10122 
10123 	spin_lock_init(&dev->addr_list_lock);
10124 	netdev_set_addr_lockdep_class(dev);
10125 
10126 	ret = dev_get_valid_name(net, dev, dev->name);
10127 	if (ret < 0)
10128 		goto out;
10129 
10130 	ret = -ENOMEM;
10131 	dev->name_node = netdev_name_node_head_alloc(dev);
10132 	if (!dev->name_node)
10133 		goto out;
10134 
10135 	/* Init, if this function is available */
10136 	if (dev->netdev_ops->ndo_init) {
10137 		ret = dev->netdev_ops->ndo_init(dev);
10138 		if (ret) {
10139 			if (ret > 0)
10140 				ret = -EIO;
10141 			goto err_free_name;
10142 		}
10143 	}
10144 
10145 	if (((dev->hw_features | dev->features) &
10146 	     NETIF_F_HW_VLAN_CTAG_FILTER) &&
10147 	    (!dev->netdev_ops->ndo_vlan_rx_add_vid ||
10148 	     !dev->netdev_ops->ndo_vlan_rx_kill_vid)) {
10149 		netdev_WARN(dev, "Buggy VLAN acceleration in driver!\n");
10150 		ret = -EINVAL;
10151 		goto err_uninit;
10152 	}
10153 
10154 	ret = -EBUSY;
10155 	if (!dev->ifindex)
10156 		dev->ifindex = dev_new_index(net);
10157 	else if (__dev_get_by_index(net, dev->ifindex))
10158 		goto err_uninit;
10159 
10160 	/* Transfer changeable features to wanted_features and enable
10161 	 * software offloads (GSO and GRO).
10162 	 */
10163 	dev->hw_features |= (NETIF_F_SOFT_FEATURES | NETIF_F_SOFT_FEATURES_OFF);
10164 	dev->features |= NETIF_F_SOFT_FEATURES;
10165 
10166 	if (dev->udp_tunnel_nic_info) {
10167 		dev->features |= NETIF_F_RX_UDP_TUNNEL_PORT;
10168 		dev->hw_features |= NETIF_F_RX_UDP_TUNNEL_PORT;
10169 	}
10170 
10171 	dev->wanted_features = dev->features & dev->hw_features;
10172 
10173 	if (!(dev->flags & IFF_LOOPBACK))
10174 		dev->hw_features |= NETIF_F_NOCACHE_COPY;
10175 
10176 	/* If IPv4 TCP segmentation offload is supported we should also
10177 	 * allow the device to enable segmenting the frame with the option
10178 	 * of ignoring a static IP ID value.  This doesn't enable the
10179 	 * feature itself but allows the user to enable it later.
10180 	 */
10181 	if (dev->hw_features & NETIF_F_TSO)
10182 		dev->hw_features |= NETIF_F_TSO_MANGLEID;
10183 	if (dev->vlan_features & NETIF_F_TSO)
10184 		dev->vlan_features |= NETIF_F_TSO_MANGLEID;
10185 	if (dev->mpls_features & NETIF_F_TSO)
10186 		dev->mpls_features |= NETIF_F_TSO_MANGLEID;
10187 	if (dev->hw_enc_features & NETIF_F_TSO)
10188 		dev->hw_enc_features |= NETIF_F_TSO_MANGLEID;
10189 
10190 	/* Make NETIF_F_HIGHDMA inheritable to VLAN devices.
10191 	 */
10192 	dev->vlan_features |= NETIF_F_HIGHDMA;
10193 
10194 	/* Make NETIF_F_SG inheritable to tunnel devices.
10195 	 */
10196 	dev->hw_enc_features |= NETIF_F_SG | NETIF_F_GSO_PARTIAL;
10197 
10198 	/* Make NETIF_F_SG inheritable to MPLS.
10199 	 */
10200 	dev->mpls_features |= NETIF_F_SG;
10201 
10202 	ret = call_netdevice_notifiers(NETDEV_POST_INIT, dev);
10203 	ret = notifier_to_errno(ret);
10204 	if (ret)
10205 		goto err_uninit;
10206 
10207 	ret = netdev_register_kobject(dev);
10208 	if (ret) {
10209 		dev->reg_state = NETREG_UNREGISTERED;
10210 		goto err_uninit;
10211 	}
10212 	dev->reg_state = NETREG_REGISTERED;
10213 
10214 	__netdev_update_features(dev);
10215 
10216 	/*
10217 	 *	Default initial state at registry is that the
10218 	 *	device is present.
10219 	 */
10220 
10221 	set_bit(__LINK_STATE_PRESENT, &dev->state);
10222 
10223 	linkwatch_init_dev(dev);
10224 
10225 	dev_init_scheduler(dev);
10226 	dev_hold(dev);
10227 	list_netdevice(dev);
10228 	add_device_randomness(dev->dev_addr, dev->addr_len);
10229 
10230 	/* If the device has permanent device address, driver should
10231 	 * set dev_addr and also addr_assign_type should be set to
10232 	 * NET_ADDR_PERM (default value).
10233 	 */
10234 	if (dev->addr_assign_type == NET_ADDR_PERM)
10235 		memcpy(dev->perm_addr, dev->dev_addr, dev->addr_len);
10236 
10237 	/* Notify protocols, that a new device appeared. */
10238 	ret = call_netdevice_notifiers(NETDEV_REGISTER, dev);
10239 	ret = notifier_to_errno(ret);
10240 	if (ret) {
10241 		/* Expect explicit free_netdev() on failure */
10242 		dev->needs_free_netdev = false;
10243 		unregister_netdevice_queue(dev, NULL);
10244 		goto out;
10245 	}
10246 	/*
10247 	 *	Prevent userspace races by waiting until the network
10248 	 *	device is fully setup before sending notifications.
10249 	 */
10250 	if (!dev->rtnl_link_ops ||
10251 	    dev->rtnl_link_state == RTNL_LINK_INITIALIZED)
10252 		rtmsg_ifinfo(RTM_NEWLINK, dev, ~0U, GFP_KERNEL);
10253 
10254 out:
10255 	return ret;
10256 
10257 err_uninit:
10258 	if (dev->netdev_ops->ndo_uninit)
10259 		dev->netdev_ops->ndo_uninit(dev);
10260 	if (dev->priv_destructor)
10261 		dev->priv_destructor(dev);
10262 err_free_name:
10263 	netdev_name_node_free(dev->name_node);
10264 	goto out;
10265 }
10266 EXPORT_SYMBOL(register_netdevice);
10267 
10268 /**
10269  *	init_dummy_netdev	- init a dummy network device for NAPI
10270  *	@dev: device to init
10271  *
10272  *	This takes a network device structure and initialize the minimum
10273  *	amount of fields so it can be used to schedule NAPI polls without
10274  *	registering a full blown interface. This is to be used by drivers
10275  *	that need to tie several hardware interfaces to a single NAPI
10276  *	poll scheduler due to HW limitations.
10277  */
10278 int init_dummy_netdev(struct net_device *dev)
10279 {
10280 	/* Clear everything. Note we don't initialize spinlocks
10281 	 * are they aren't supposed to be taken by any of the
10282 	 * NAPI code and this dummy netdev is supposed to be
10283 	 * only ever used for NAPI polls
10284 	 */
10285 	memset(dev, 0, sizeof(struct net_device));
10286 
10287 	/* make sure we BUG if trying to hit standard
10288 	 * register/unregister code path
10289 	 */
10290 	dev->reg_state = NETREG_DUMMY;
10291 
10292 	/* NAPI wants this */
10293 	INIT_LIST_HEAD(&dev->napi_list);
10294 
10295 	/* a dummy interface is started by default */
10296 	set_bit(__LINK_STATE_PRESENT, &dev->state);
10297 	set_bit(__LINK_STATE_START, &dev->state);
10298 
10299 	/* napi_busy_loop stats accounting wants this */
10300 	dev_net_set(dev, &init_net);
10301 
10302 	/* Note : We dont allocate pcpu_refcnt for dummy devices,
10303 	 * because users of this 'device' dont need to change
10304 	 * its refcount.
10305 	 */
10306 
10307 	return 0;
10308 }
10309 EXPORT_SYMBOL_GPL(init_dummy_netdev);
10310 
10311 
10312 /**
10313  *	register_netdev	- register a network device
10314  *	@dev: device to register
10315  *
10316  *	Take a completed network device structure and add it to the kernel
10317  *	interfaces. A %NETDEV_REGISTER message is sent to the netdev notifier
10318  *	chain. 0 is returned on success. A negative errno code is returned
10319  *	on a failure to set up the device, or if the name is a duplicate.
10320  *
10321  *	This is a wrapper around register_netdevice that takes the rtnl semaphore
10322  *	and expands the device name if you passed a format string to
10323  *	alloc_netdev.
10324  */
10325 int register_netdev(struct net_device *dev)
10326 {
10327 	int err;
10328 
10329 	if (rtnl_lock_killable())
10330 		return -EINTR;
10331 	err = register_netdevice(dev);
10332 	rtnl_unlock();
10333 	return err;
10334 }
10335 EXPORT_SYMBOL(register_netdev);
10336 
10337 int netdev_refcnt_read(const struct net_device *dev)
10338 {
10339 	int i, refcnt = 0;
10340 
10341 	for_each_possible_cpu(i)
10342 		refcnt += *per_cpu_ptr(dev->pcpu_refcnt, i);
10343 	return refcnt;
10344 }
10345 EXPORT_SYMBOL(netdev_refcnt_read);
10346 
10347 #define WAIT_REFS_MIN_MSECS 1
10348 #define WAIT_REFS_MAX_MSECS 250
10349 /**
10350  * netdev_wait_allrefs - wait until all references are gone.
10351  * @dev: target net_device
10352  *
10353  * This is called when unregistering network devices.
10354  *
10355  * Any protocol or device that holds a reference should register
10356  * for netdevice notification, and cleanup and put back the
10357  * reference if they receive an UNREGISTER event.
10358  * We can get stuck here if buggy protocols don't correctly
10359  * call dev_put.
10360  */
10361 static void netdev_wait_allrefs(struct net_device *dev)
10362 {
10363 	unsigned long rebroadcast_time, warning_time;
10364 	int wait = 0, refcnt;
10365 
10366 	linkwatch_forget_dev(dev);
10367 
10368 	rebroadcast_time = warning_time = jiffies;
10369 	refcnt = netdev_refcnt_read(dev);
10370 
10371 	while (refcnt != 0) {
10372 		if (time_after(jiffies, rebroadcast_time + 1 * HZ)) {
10373 			rtnl_lock();
10374 
10375 			/* Rebroadcast unregister notification */
10376 			call_netdevice_notifiers(NETDEV_UNREGISTER, dev);
10377 
10378 			__rtnl_unlock();
10379 			rcu_barrier();
10380 			rtnl_lock();
10381 
10382 			if (test_bit(__LINK_STATE_LINKWATCH_PENDING,
10383 				     &dev->state)) {
10384 				/* We must not have linkwatch events
10385 				 * pending on unregister. If this
10386 				 * happens, we simply run the queue
10387 				 * unscheduled, resulting in a noop
10388 				 * for this device.
10389 				 */
10390 				linkwatch_run_queue();
10391 			}
10392 
10393 			__rtnl_unlock();
10394 
10395 			rebroadcast_time = jiffies;
10396 		}
10397 
10398 		if (!wait) {
10399 			rcu_barrier();
10400 			wait = WAIT_REFS_MIN_MSECS;
10401 		} else {
10402 			msleep(wait);
10403 			wait = min(wait << 1, WAIT_REFS_MAX_MSECS);
10404 		}
10405 
10406 		refcnt = netdev_refcnt_read(dev);
10407 
10408 		if (refcnt && time_after(jiffies, warning_time + 10 * HZ)) {
10409 			pr_emerg("unregister_netdevice: waiting for %s to become free. Usage count = %d\n",
10410 				 dev->name, refcnt);
10411 			warning_time = jiffies;
10412 		}
10413 	}
10414 }
10415 
10416 /* The sequence is:
10417  *
10418  *	rtnl_lock();
10419  *	...
10420  *	register_netdevice(x1);
10421  *	register_netdevice(x2);
10422  *	...
10423  *	unregister_netdevice(y1);
10424  *	unregister_netdevice(y2);
10425  *      ...
10426  *	rtnl_unlock();
10427  *	free_netdev(y1);
10428  *	free_netdev(y2);
10429  *
10430  * We are invoked by rtnl_unlock().
10431  * This allows us to deal with problems:
10432  * 1) We can delete sysfs objects which invoke hotplug
10433  *    without deadlocking with linkwatch via keventd.
10434  * 2) Since we run with the RTNL semaphore not held, we can sleep
10435  *    safely in order to wait for the netdev refcnt to drop to zero.
10436  *
10437  * We must not return until all unregister events added during
10438  * the interval the lock was held have been completed.
10439  */
10440 void netdev_run_todo(void)
10441 {
10442 	struct list_head list;
10443 #ifdef CONFIG_LOCKDEP
10444 	struct list_head unlink_list;
10445 
10446 	list_replace_init(&net_unlink_list, &unlink_list);
10447 
10448 	while (!list_empty(&unlink_list)) {
10449 		struct net_device *dev = list_first_entry(&unlink_list,
10450 							  struct net_device,
10451 							  unlink_list);
10452 		list_del_init(&dev->unlink_list);
10453 		dev->nested_level = dev->lower_level - 1;
10454 	}
10455 #endif
10456 
10457 	/* Snapshot list, allow later requests */
10458 	list_replace_init(&net_todo_list, &list);
10459 
10460 	__rtnl_unlock();
10461 
10462 
10463 	/* Wait for rcu callbacks to finish before next phase */
10464 	if (!list_empty(&list))
10465 		rcu_barrier();
10466 
10467 	while (!list_empty(&list)) {
10468 		struct net_device *dev
10469 			= list_first_entry(&list, struct net_device, todo_list);
10470 		list_del(&dev->todo_list);
10471 
10472 		if (unlikely(dev->reg_state != NETREG_UNREGISTERING)) {
10473 			pr_err("network todo '%s' but state %d\n",
10474 			       dev->name, dev->reg_state);
10475 			dump_stack();
10476 			continue;
10477 		}
10478 
10479 		dev->reg_state = NETREG_UNREGISTERED;
10480 
10481 		netdev_wait_allrefs(dev);
10482 
10483 		/* paranoia */
10484 		BUG_ON(netdev_refcnt_read(dev));
10485 		BUG_ON(!list_empty(&dev->ptype_all));
10486 		BUG_ON(!list_empty(&dev->ptype_specific));
10487 		WARN_ON(rcu_access_pointer(dev->ip_ptr));
10488 		WARN_ON(rcu_access_pointer(dev->ip6_ptr));
10489 #if IS_ENABLED(CONFIG_DECNET)
10490 		WARN_ON(dev->dn_ptr);
10491 #endif
10492 		if (dev->priv_destructor)
10493 			dev->priv_destructor(dev);
10494 		if (dev->needs_free_netdev)
10495 			free_netdev(dev);
10496 
10497 		/* Report a network device has been unregistered */
10498 		rtnl_lock();
10499 		dev_net(dev)->dev_unreg_count--;
10500 		__rtnl_unlock();
10501 		wake_up(&netdev_unregistering_wq);
10502 
10503 		/* Free network device */
10504 		kobject_put(&dev->dev.kobj);
10505 	}
10506 }
10507 
10508 /* Convert net_device_stats to rtnl_link_stats64. rtnl_link_stats64 has
10509  * all the same fields in the same order as net_device_stats, with only
10510  * the type differing, but rtnl_link_stats64 may have additional fields
10511  * at the end for newer counters.
10512  */
10513 void netdev_stats_to_stats64(struct rtnl_link_stats64 *stats64,
10514 			     const struct net_device_stats *netdev_stats)
10515 {
10516 #if BITS_PER_LONG == 64
10517 	BUILD_BUG_ON(sizeof(*stats64) < sizeof(*netdev_stats));
10518 	memcpy(stats64, netdev_stats, sizeof(*netdev_stats));
10519 	/* zero out counters that only exist in rtnl_link_stats64 */
10520 	memset((char *)stats64 + sizeof(*netdev_stats), 0,
10521 	       sizeof(*stats64) - sizeof(*netdev_stats));
10522 #else
10523 	size_t i, n = sizeof(*netdev_stats) / sizeof(unsigned long);
10524 	const unsigned long *src = (const unsigned long *)netdev_stats;
10525 	u64 *dst = (u64 *)stats64;
10526 
10527 	BUILD_BUG_ON(n > sizeof(*stats64) / sizeof(u64));
10528 	for (i = 0; i < n; i++)
10529 		dst[i] = src[i];
10530 	/* zero out counters that only exist in rtnl_link_stats64 */
10531 	memset((char *)stats64 + n * sizeof(u64), 0,
10532 	       sizeof(*stats64) - n * sizeof(u64));
10533 #endif
10534 }
10535 EXPORT_SYMBOL(netdev_stats_to_stats64);
10536 
10537 /**
10538  *	dev_get_stats	- get network device statistics
10539  *	@dev: device to get statistics from
10540  *	@storage: place to store stats
10541  *
10542  *	Get network statistics from device. Return @storage.
10543  *	The device driver may provide its own method by setting
10544  *	dev->netdev_ops->get_stats64 or dev->netdev_ops->get_stats;
10545  *	otherwise the internal statistics structure is used.
10546  */
10547 struct rtnl_link_stats64 *dev_get_stats(struct net_device *dev,
10548 					struct rtnl_link_stats64 *storage)
10549 {
10550 	const struct net_device_ops *ops = dev->netdev_ops;
10551 
10552 	if (ops->ndo_get_stats64) {
10553 		memset(storage, 0, sizeof(*storage));
10554 		ops->ndo_get_stats64(dev, storage);
10555 	} else if (ops->ndo_get_stats) {
10556 		netdev_stats_to_stats64(storage, ops->ndo_get_stats(dev));
10557 	} else {
10558 		netdev_stats_to_stats64(storage, &dev->stats);
10559 	}
10560 	storage->rx_dropped += (unsigned long)atomic_long_read(&dev->rx_dropped);
10561 	storage->tx_dropped += (unsigned long)atomic_long_read(&dev->tx_dropped);
10562 	storage->rx_nohandler += (unsigned long)atomic_long_read(&dev->rx_nohandler);
10563 	return storage;
10564 }
10565 EXPORT_SYMBOL(dev_get_stats);
10566 
10567 /**
10568  *	dev_fetch_sw_netstats - get per-cpu network device statistics
10569  *	@s: place to store stats
10570  *	@netstats: per-cpu network stats to read from
10571  *
10572  *	Read per-cpu network statistics and populate the related fields in @s.
10573  */
10574 void dev_fetch_sw_netstats(struct rtnl_link_stats64 *s,
10575 			   const struct pcpu_sw_netstats __percpu *netstats)
10576 {
10577 	int cpu;
10578 
10579 	for_each_possible_cpu(cpu) {
10580 		const struct pcpu_sw_netstats *stats;
10581 		struct pcpu_sw_netstats tmp;
10582 		unsigned int start;
10583 
10584 		stats = per_cpu_ptr(netstats, cpu);
10585 		do {
10586 			start = u64_stats_fetch_begin_irq(&stats->syncp);
10587 			tmp.rx_packets = stats->rx_packets;
10588 			tmp.rx_bytes   = stats->rx_bytes;
10589 			tmp.tx_packets = stats->tx_packets;
10590 			tmp.tx_bytes   = stats->tx_bytes;
10591 		} while (u64_stats_fetch_retry_irq(&stats->syncp, start));
10592 
10593 		s->rx_packets += tmp.rx_packets;
10594 		s->rx_bytes   += tmp.rx_bytes;
10595 		s->tx_packets += tmp.tx_packets;
10596 		s->tx_bytes   += tmp.tx_bytes;
10597 	}
10598 }
10599 EXPORT_SYMBOL_GPL(dev_fetch_sw_netstats);
10600 
10601 /**
10602  *	dev_get_tstats64 - ndo_get_stats64 implementation
10603  *	@dev: device to get statistics from
10604  *	@s: place to store stats
10605  *
10606  *	Populate @s from dev->stats and dev->tstats. Can be used as
10607  *	ndo_get_stats64() callback.
10608  */
10609 void dev_get_tstats64(struct net_device *dev, struct rtnl_link_stats64 *s)
10610 {
10611 	netdev_stats_to_stats64(s, &dev->stats);
10612 	dev_fetch_sw_netstats(s, dev->tstats);
10613 }
10614 EXPORT_SYMBOL_GPL(dev_get_tstats64);
10615 
10616 struct netdev_queue *dev_ingress_queue_create(struct net_device *dev)
10617 {
10618 	struct netdev_queue *queue = dev_ingress_queue(dev);
10619 
10620 #ifdef CONFIG_NET_CLS_ACT
10621 	if (queue)
10622 		return queue;
10623 	queue = kzalloc(sizeof(*queue), GFP_KERNEL);
10624 	if (!queue)
10625 		return NULL;
10626 	netdev_init_one_queue(dev, queue, NULL);
10627 	RCU_INIT_POINTER(queue->qdisc, &noop_qdisc);
10628 	queue->qdisc_sleeping = &noop_qdisc;
10629 	rcu_assign_pointer(dev->ingress_queue, queue);
10630 #endif
10631 	return queue;
10632 }
10633 
10634 static const struct ethtool_ops default_ethtool_ops;
10635 
10636 void netdev_set_default_ethtool_ops(struct net_device *dev,
10637 				    const struct ethtool_ops *ops)
10638 {
10639 	if (dev->ethtool_ops == &default_ethtool_ops)
10640 		dev->ethtool_ops = ops;
10641 }
10642 EXPORT_SYMBOL_GPL(netdev_set_default_ethtool_ops);
10643 
10644 void netdev_freemem(struct net_device *dev)
10645 {
10646 	char *addr = (char *)dev - dev->padded;
10647 
10648 	kvfree(addr);
10649 }
10650 
10651 /**
10652  * alloc_netdev_mqs - allocate network device
10653  * @sizeof_priv: size of private data to allocate space for
10654  * @name: device name format string
10655  * @name_assign_type: origin of device name
10656  * @setup: callback to initialize device
10657  * @txqs: the number of TX subqueues to allocate
10658  * @rxqs: the number of RX subqueues to allocate
10659  *
10660  * Allocates a struct net_device with private data area for driver use
10661  * and performs basic initialization.  Also allocates subqueue structs
10662  * for each queue on the device.
10663  */
10664 struct net_device *alloc_netdev_mqs(int sizeof_priv, const char *name,
10665 		unsigned char name_assign_type,
10666 		void (*setup)(struct net_device *),
10667 		unsigned int txqs, unsigned int rxqs)
10668 {
10669 	struct net_device *dev;
10670 	unsigned int alloc_size;
10671 	struct net_device *p;
10672 
10673 	BUG_ON(strlen(name) >= sizeof(dev->name));
10674 
10675 	if (txqs < 1) {
10676 		pr_err("alloc_netdev: Unable to allocate device with zero queues\n");
10677 		return NULL;
10678 	}
10679 
10680 	if (rxqs < 1) {
10681 		pr_err("alloc_netdev: Unable to allocate device with zero RX queues\n");
10682 		return NULL;
10683 	}
10684 
10685 	alloc_size = sizeof(struct net_device);
10686 	if (sizeof_priv) {
10687 		/* ensure 32-byte alignment of private area */
10688 		alloc_size = ALIGN(alloc_size, NETDEV_ALIGN);
10689 		alloc_size += sizeof_priv;
10690 	}
10691 	/* ensure 32-byte alignment of whole construct */
10692 	alloc_size += NETDEV_ALIGN - 1;
10693 
10694 	p = kvzalloc(alloc_size, GFP_KERNEL | __GFP_RETRY_MAYFAIL);
10695 	if (!p)
10696 		return NULL;
10697 
10698 	dev = PTR_ALIGN(p, NETDEV_ALIGN);
10699 	dev->padded = (char *)dev - (char *)p;
10700 
10701 	dev->pcpu_refcnt = alloc_percpu(int);
10702 	if (!dev->pcpu_refcnt)
10703 		goto free_dev;
10704 
10705 	if (dev_addr_init(dev))
10706 		goto free_pcpu;
10707 
10708 	dev_mc_init(dev);
10709 	dev_uc_init(dev);
10710 
10711 	dev_net_set(dev, &init_net);
10712 
10713 	dev->gso_max_size = GSO_MAX_SIZE;
10714 	dev->gso_max_segs = GSO_MAX_SEGS;
10715 	dev->upper_level = 1;
10716 	dev->lower_level = 1;
10717 #ifdef CONFIG_LOCKDEP
10718 	dev->nested_level = 0;
10719 	INIT_LIST_HEAD(&dev->unlink_list);
10720 #endif
10721 
10722 	INIT_LIST_HEAD(&dev->napi_list);
10723 	INIT_LIST_HEAD(&dev->unreg_list);
10724 	INIT_LIST_HEAD(&dev->close_list);
10725 	INIT_LIST_HEAD(&dev->link_watch_list);
10726 	INIT_LIST_HEAD(&dev->adj_list.upper);
10727 	INIT_LIST_HEAD(&dev->adj_list.lower);
10728 	INIT_LIST_HEAD(&dev->ptype_all);
10729 	INIT_LIST_HEAD(&dev->ptype_specific);
10730 	INIT_LIST_HEAD(&dev->net_notifier_list);
10731 #ifdef CONFIG_NET_SCHED
10732 	hash_init(dev->qdisc_hash);
10733 #endif
10734 	dev->priv_flags = IFF_XMIT_DST_RELEASE | IFF_XMIT_DST_RELEASE_PERM;
10735 	setup(dev);
10736 
10737 	if (!dev->tx_queue_len) {
10738 		dev->priv_flags |= IFF_NO_QUEUE;
10739 		dev->tx_queue_len = DEFAULT_TX_QUEUE_LEN;
10740 	}
10741 
10742 	dev->num_tx_queues = txqs;
10743 	dev->real_num_tx_queues = txqs;
10744 	if (netif_alloc_netdev_queues(dev))
10745 		goto free_all;
10746 
10747 	dev->num_rx_queues = rxqs;
10748 	dev->real_num_rx_queues = rxqs;
10749 	if (netif_alloc_rx_queues(dev))
10750 		goto free_all;
10751 
10752 	strcpy(dev->name, name);
10753 	dev->name_assign_type = name_assign_type;
10754 	dev->group = INIT_NETDEV_GROUP;
10755 	if (!dev->ethtool_ops)
10756 		dev->ethtool_ops = &default_ethtool_ops;
10757 
10758 	nf_hook_ingress_init(dev);
10759 
10760 	return dev;
10761 
10762 free_all:
10763 	free_netdev(dev);
10764 	return NULL;
10765 
10766 free_pcpu:
10767 	free_percpu(dev->pcpu_refcnt);
10768 free_dev:
10769 	netdev_freemem(dev);
10770 	return NULL;
10771 }
10772 EXPORT_SYMBOL(alloc_netdev_mqs);
10773 
10774 /**
10775  * free_netdev - free network device
10776  * @dev: device
10777  *
10778  * This function does the last stage of destroying an allocated device
10779  * interface. The reference to the device object is released. If this
10780  * is the last reference then it will be freed.Must be called in process
10781  * context.
10782  */
10783 void free_netdev(struct net_device *dev)
10784 {
10785 	struct napi_struct *p, *n;
10786 
10787 	might_sleep();
10788 
10789 	/* When called immediately after register_netdevice() failed the unwind
10790 	 * handling may still be dismantling the device. Handle that case by
10791 	 * deferring the free.
10792 	 */
10793 	if (dev->reg_state == NETREG_UNREGISTERING) {
10794 		ASSERT_RTNL();
10795 		dev->needs_free_netdev = true;
10796 		return;
10797 	}
10798 
10799 	netif_free_tx_queues(dev);
10800 	netif_free_rx_queues(dev);
10801 
10802 	kfree(rcu_dereference_protected(dev->ingress_queue, 1));
10803 
10804 	/* Flush device addresses */
10805 	dev_addr_flush(dev);
10806 
10807 	list_for_each_entry_safe(p, n, &dev->napi_list, dev_list)
10808 		netif_napi_del(p);
10809 
10810 	free_percpu(dev->pcpu_refcnt);
10811 	dev->pcpu_refcnt = NULL;
10812 	free_percpu(dev->xdp_bulkq);
10813 	dev->xdp_bulkq = NULL;
10814 
10815 	/*  Compatibility with error handling in drivers */
10816 	if (dev->reg_state == NETREG_UNINITIALIZED) {
10817 		netdev_freemem(dev);
10818 		return;
10819 	}
10820 
10821 	BUG_ON(dev->reg_state != NETREG_UNREGISTERED);
10822 	dev->reg_state = NETREG_RELEASED;
10823 
10824 	/* will free via device release */
10825 	put_device(&dev->dev);
10826 }
10827 EXPORT_SYMBOL(free_netdev);
10828 
10829 /**
10830  *	synchronize_net -  Synchronize with packet receive processing
10831  *
10832  *	Wait for packets currently being received to be done.
10833  *	Does not block later packets from starting.
10834  */
10835 void synchronize_net(void)
10836 {
10837 	might_sleep();
10838 	if (rtnl_is_locked())
10839 		synchronize_rcu_expedited();
10840 	else
10841 		synchronize_rcu();
10842 }
10843 EXPORT_SYMBOL(synchronize_net);
10844 
10845 /**
10846  *	unregister_netdevice_queue - remove device from the kernel
10847  *	@dev: device
10848  *	@head: list
10849  *
10850  *	This function shuts down a device interface and removes it
10851  *	from the kernel tables.
10852  *	If head not NULL, device is queued to be unregistered later.
10853  *
10854  *	Callers must hold the rtnl semaphore.  You may want
10855  *	unregister_netdev() instead of this.
10856  */
10857 
10858 void unregister_netdevice_queue(struct net_device *dev, struct list_head *head)
10859 {
10860 	ASSERT_RTNL();
10861 
10862 	if (head) {
10863 		list_move_tail(&dev->unreg_list, head);
10864 	} else {
10865 		LIST_HEAD(single);
10866 
10867 		list_add(&dev->unreg_list, &single);
10868 		unregister_netdevice_many(&single);
10869 	}
10870 }
10871 EXPORT_SYMBOL(unregister_netdevice_queue);
10872 
10873 /**
10874  *	unregister_netdevice_many - unregister many devices
10875  *	@head: list of devices
10876  *
10877  *  Note: As most callers use a stack allocated list_head,
10878  *  we force a list_del() to make sure stack wont be corrupted later.
10879  */
10880 void unregister_netdevice_many(struct list_head *head)
10881 {
10882 	struct net_device *dev, *tmp;
10883 	LIST_HEAD(close_head);
10884 
10885 	BUG_ON(dev_boot_phase);
10886 	ASSERT_RTNL();
10887 
10888 	if (list_empty(head))
10889 		return;
10890 
10891 	list_for_each_entry_safe(dev, tmp, head, unreg_list) {
10892 		/* Some devices call without registering
10893 		 * for initialization unwind. Remove those
10894 		 * devices and proceed with the remaining.
10895 		 */
10896 		if (dev->reg_state == NETREG_UNINITIALIZED) {
10897 			pr_debug("unregister_netdevice: device %s/%p never was registered\n",
10898 				 dev->name, dev);
10899 
10900 			WARN_ON(1);
10901 			list_del(&dev->unreg_list);
10902 			continue;
10903 		}
10904 		dev->dismantle = true;
10905 		BUG_ON(dev->reg_state != NETREG_REGISTERED);
10906 	}
10907 
10908 	/* If device is running, close it first. */
10909 	list_for_each_entry(dev, head, unreg_list)
10910 		list_add_tail(&dev->close_list, &close_head);
10911 	dev_close_many(&close_head, true);
10912 
10913 	list_for_each_entry(dev, head, unreg_list) {
10914 		/* And unlink it from device chain. */
10915 		unlist_netdevice(dev);
10916 
10917 		dev->reg_state = NETREG_UNREGISTERING;
10918 	}
10919 	flush_all_backlogs();
10920 
10921 	synchronize_net();
10922 
10923 	list_for_each_entry(dev, head, unreg_list) {
10924 		struct sk_buff *skb = NULL;
10925 
10926 		/* Shutdown queueing discipline. */
10927 		dev_shutdown(dev);
10928 
10929 		dev_xdp_uninstall(dev);
10930 
10931 		/* Notify protocols, that we are about to destroy
10932 		 * this device. They should clean all the things.
10933 		 */
10934 		call_netdevice_notifiers(NETDEV_UNREGISTER, dev);
10935 
10936 		if (!dev->rtnl_link_ops ||
10937 		    dev->rtnl_link_state == RTNL_LINK_INITIALIZED)
10938 			skb = rtmsg_ifinfo_build_skb(RTM_DELLINK, dev, ~0U, 0,
10939 						     GFP_KERNEL, NULL, 0);
10940 
10941 		/*
10942 		 *	Flush the unicast and multicast chains
10943 		 */
10944 		dev_uc_flush(dev);
10945 		dev_mc_flush(dev);
10946 
10947 		netdev_name_node_alt_flush(dev);
10948 		netdev_name_node_free(dev->name_node);
10949 
10950 		if (dev->netdev_ops->ndo_uninit)
10951 			dev->netdev_ops->ndo_uninit(dev);
10952 
10953 		if (skb)
10954 			rtmsg_ifinfo_send(skb, dev, GFP_KERNEL);
10955 
10956 		/* Notifier chain MUST detach us all upper devices. */
10957 		WARN_ON(netdev_has_any_upper_dev(dev));
10958 		WARN_ON(netdev_has_any_lower_dev(dev));
10959 
10960 		/* Remove entries from kobject tree */
10961 		netdev_unregister_kobject(dev);
10962 #ifdef CONFIG_XPS
10963 		/* Remove XPS queueing entries */
10964 		netif_reset_xps_queues_gt(dev, 0);
10965 #endif
10966 	}
10967 
10968 	synchronize_net();
10969 
10970 	list_for_each_entry(dev, head, unreg_list) {
10971 		dev_put(dev);
10972 		net_set_todo(dev);
10973 	}
10974 
10975 	list_del(head);
10976 }
10977 EXPORT_SYMBOL(unregister_netdevice_many);
10978 
10979 /**
10980  *	unregister_netdev - remove device from the kernel
10981  *	@dev: device
10982  *
10983  *	This function shuts down a device interface and removes it
10984  *	from the kernel tables.
10985  *
10986  *	This is just a wrapper for unregister_netdevice that takes
10987  *	the rtnl semaphore.  In general you want to use this and not
10988  *	unregister_netdevice.
10989  */
10990 void unregister_netdev(struct net_device *dev)
10991 {
10992 	rtnl_lock();
10993 	unregister_netdevice(dev);
10994 	rtnl_unlock();
10995 }
10996 EXPORT_SYMBOL(unregister_netdev);
10997 
10998 /**
10999  *	dev_change_net_namespace - move device to different nethost namespace
11000  *	@dev: device
11001  *	@net: network namespace
11002  *	@pat: If not NULL name pattern to try if the current device name
11003  *	      is already taken in the destination network namespace.
11004  *
11005  *	This function shuts down a device interface and moves it
11006  *	to a new network namespace. On success 0 is returned, on
11007  *	a failure a netagive errno code is returned.
11008  *
11009  *	Callers must hold the rtnl semaphore.
11010  */
11011 
11012 int dev_change_net_namespace(struct net_device *dev, struct net *net, const char *pat)
11013 {
11014 	struct net *net_old = dev_net(dev);
11015 	int err, new_nsid, new_ifindex;
11016 
11017 	ASSERT_RTNL();
11018 
11019 	/* Don't allow namespace local devices to be moved. */
11020 	err = -EINVAL;
11021 	if (dev->features & NETIF_F_NETNS_LOCAL)
11022 		goto out;
11023 
11024 	/* Ensure the device has been registrered */
11025 	if (dev->reg_state != NETREG_REGISTERED)
11026 		goto out;
11027 
11028 	/* Get out if there is nothing todo */
11029 	err = 0;
11030 	if (net_eq(net_old, net))
11031 		goto out;
11032 
11033 	/* Pick the destination device name, and ensure
11034 	 * we can use it in the destination network namespace.
11035 	 */
11036 	err = -EEXIST;
11037 	if (__dev_get_by_name(net, dev->name)) {
11038 		/* We get here if we can't use the current device name */
11039 		if (!pat)
11040 			goto out;
11041 		err = dev_get_valid_name(net, dev, pat);
11042 		if (err < 0)
11043 			goto out;
11044 	}
11045 
11046 	/*
11047 	 * And now a mini version of register_netdevice unregister_netdevice.
11048 	 */
11049 
11050 	/* If device is running close it first. */
11051 	dev_close(dev);
11052 
11053 	/* And unlink it from device chain */
11054 	unlist_netdevice(dev);
11055 
11056 	synchronize_net();
11057 
11058 	/* Shutdown queueing discipline. */
11059 	dev_shutdown(dev);
11060 
11061 	/* Notify protocols, that we are about to destroy
11062 	 * this device. They should clean all the things.
11063 	 *
11064 	 * Note that dev->reg_state stays at NETREG_REGISTERED.
11065 	 * This is wanted because this way 8021q and macvlan know
11066 	 * the device is just moving and can keep their slaves up.
11067 	 */
11068 	call_netdevice_notifiers(NETDEV_UNREGISTER, dev);
11069 	rcu_barrier();
11070 
11071 	new_nsid = peernet2id_alloc(dev_net(dev), net, GFP_KERNEL);
11072 	/* If there is an ifindex conflict assign a new one */
11073 	if (__dev_get_by_index(net, dev->ifindex))
11074 		new_ifindex = dev_new_index(net);
11075 	else
11076 		new_ifindex = dev->ifindex;
11077 
11078 	rtmsg_ifinfo_newnet(RTM_DELLINK, dev, ~0U, GFP_KERNEL, &new_nsid,
11079 			    new_ifindex);
11080 
11081 	/*
11082 	 *	Flush the unicast and multicast chains
11083 	 */
11084 	dev_uc_flush(dev);
11085 	dev_mc_flush(dev);
11086 
11087 	/* Send a netdev-removed uevent to the old namespace */
11088 	kobject_uevent(&dev->dev.kobj, KOBJ_REMOVE);
11089 	netdev_adjacent_del_links(dev);
11090 
11091 	/* Move per-net netdevice notifiers that are following the netdevice */
11092 	move_netdevice_notifiers_dev_net(dev, net);
11093 
11094 	/* Actually switch the network namespace */
11095 	dev_net_set(dev, net);
11096 	dev->ifindex = new_ifindex;
11097 
11098 	/* Send a netdev-add uevent to the new namespace */
11099 	kobject_uevent(&dev->dev.kobj, KOBJ_ADD);
11100 	netdev_adjacent_add_links(dev);
11101 
11102 	/* Fixup kobjects */
11103 	err = device_rename(&dev->dev, dev->name);
11104 	WARN_ON(err);
11105 
11106 	/* Adapt owner in case owning user namespace of target network
11107 	 * namespace is different from the original one.
11108 	 */
11109 	err = netdev_change_owner(dev, net_old, net);
11110 	WARN_ON(err);
11111 
11112 	/* Add the device back in the hashes */
11113 	list_netdevice(dev);
11114 
11115 	/* Notify protocols, that a new device appeared. */
11116 	call_netdevice_notifiers(NETDEV_REGISTER, dev);
11117 
11118 	/*
11119 	 *	Prevent userspace races by waiting until the network
11120 	 *	device is fully setup before sending notifications.
11121 	 */
11122 	rtmsg_ifinfo(RTM_NEWLINK, dev, ~0U, GFP_KERNEL);
11123 
11124 	synchronize_net();
11125 	err = 0;
11126 out:
11127 	return err;
11128 }
11129 EXPORT_SYMBOL_GPL(dev_change_net_namespace);
11130 
11131 static int dev_cpu_dead(unsigned int oldcpu)
11132 {
11133 	struct sk_buff **list_skb;
11134 	struct sk_buff *skb;
11135 	unsigned int cpu;
11136 	struct softnet_data *sd, *oldsd, *remsd = NULL;
11137 
11138 	local_irq_disable();
11139 	cpu = smp_processor_id();
11140 	sd = &per_cpu(softnet_data, cpu);
11141 	oldsd = &per_cpu(softnet_data, oldcpu);
11142 
11143 	/* Find end of our completion_queue. */
11144 	list_skb = &sd->completion_queue;
11145 	while (*list_skb)
11146 		list_skb = &(*list_skb)->next;
11147 	/* Append completion queue from offline CPU. */
11148 	*list_skb = oldsd->completion_queue;
11149 	oldsd->completion_queue = NULL;
11150 
11151 	/* Append output queue from offline CPU. */
11152 	if (oldsd->output_queue) {
11153 		*sd->output_queue_tailp = oldsd->output_queue;
11154 		sd->output_queue_tailp = oldsd->output_queue_tailp;
11155 		oldsd->output_queue = NULL;
11156 		oldsd->output_queue_tailp = &oldsd->output_queue;
11157 	}
11158 	/* Append NAPI poll list from offline CPU, with one exception :
11159 	 * process_backlog() must be called by cpu owning percpu backlog.
11160 	 * We properly handle process_queue & input_pkt_queue later.
11161 	 */
11162 	while (!list_empty(&oldsd->poll_list)) {
11163 		struct napi_struct *napi = list_first_entry(&oldsd->poll_list,
11164 							    struct napi_struct,
11165 							    poll_list);
11166 
11167 		list_del_init(&napi->poll_list);
11168 		if (napi->poll == process_backlog)
11169 			napi->state = 0;
11170 		else
11171 			____napi_schedule(sd, napi);
11172 	}
11173 
11174 	raise_softirq_irqoff(NET_TX_SOFTIRQ);
11175 	local_irq_enable();
11176 
11177 #ifdef CONFIG_RPS
11178 	remsd = oldsd->rps_ipi_list;
11179 	oldsd->rps_ipi_list = NULL;
11180 #endif
11181 	/* send out pending IPI's on offline CPU */
11182 	net_rps_send_ipi(remsd);
11183 
11184 	/* Process offline CPU's input_pkt_queue */
11185 	while ((skb = __skb_dequeue(&oldsd->process_queue))) {
11186 		netif_rx_ni(skb);
11187 		input_queue_head_incr(oldsd);
11188 	}
11189 	while ((skb = skb_dequeue(&oldsd->input_pkt_queue))) {
11190 		netif_rx_ni(skb);
11191 		input_queue_head_incr(oldsd);
11192 	}
11193 
11194 	return 0;
11195 }
11196 
11197 /**
11198  *	netdev_increment_features - increment feature set by one
11199  *	@all: current feature set
11200  *	@one: new feature set
11201  *	@mask: mask feature set
11202  *
11203  *	Computes a new feature set after adding a device with feature set
11204  *	@one to the master device with current feature set @all.  Will not
11205  *	enable anything that is off in @mask. Returns the new feature set.
11206  */
11207 netdev_features_t netdev_increment_features(netdev_features_t all,
11208 	netdev_features_t one, netdev_features_t mask)
11209 {
11210 	if (mask & NETIF_F_HW_CSUM)
11211 		mask |= NETIF_F_CSUM_MASK;
11212 	mask |= NETIF_F_VLAN_CHALLENGED;
11213 
11214 	all |= one & (NETIF_F_ONE_FOR_ALL | NETIF_F_CSUM_MASK) & mask;
11215 	all &= one | ~NETIF_F_ALL_FOR_ALL;
11216 
11217 	/* If one device supports hw checksumming, set for all. */
11218 	if (all & NETIF_F_HW_CSUM)
11219 		all &= ~(NETIF_F_CSUM_MASK & ~NETIF_F_HW_CSUM);
11220 
11221 	return all;
11222 }
11223 EXPORT_SYMBOL(netdev_increment_features);
11224 
11225 static struct hlist_head * __net_init netdev_create_hash(void)
11226 {
11227 	int i;
11228 	struct hlist_head *hash;
11229 
11230 	hash = kmalloc_array(NETDEV_HASHENTRIES, sizeof(*hash), GFP_KERNEL);
11231 	if (hash != NULL)
11232 		for (i = 0; i < NETDEV_HASHENTRIES; i++)
11233 			INIT_HLIST_HEAD(&hash[i]);
11234 
11235 	return hash;
11236 }
11237 
11238 /* Initialize per network namespace state */
11239 static int __net_init netdev_init(struct net *net)
11240 {
11241 	BUILD_BUG_ON(GRO_HASH_BUCKETS >
11242 		     8 * sizeof_field(struct napi_struct, gro_bitmask));
11243 
11244 	if (net != &init_net)
11245 		INIT_LIST_HEAD(&net->dev_base_head);
11246 
11247 	net->dev_name_head = netdev_create_hash();
11248 	if (net->dev_name_head == NULL)
11249 		goto err_name;
11250 
11251 	net->dev_index_head = netdev_create_hash();
11252 	if (net->dev_index_head == NULL)
11253 		goto err_idx;
11254 
11255 	RAW_INIT_NOTIFIER_HEAD(&net->netdev_chain);
11256 
11257 	return 0;
11258 
11259 err_idx:
11260 	kfree(net->dev_name_head);
11261 err_name:
11262 	return -ENOMEM;
11263 }
11264 
11265 /**
11266  *	netdev_drivername - network driver for the device
11267  *	@dev: network device
11268  *
11269  *	Determine network driver for device.
11270  */
11271 const char *netdev_drivername(const struct net_device *dev)
11272 {
11273 	const struct device_driver *driver;
11274 	const struct device *parent;
11275 	const char *empty = "";
11276 
11277 	parent = dev->dev.parent;
11278 	if (!parent)
11279 		return empty;
11280 
11281 	driver = parent->driver;
11282 	if (driver && driver->name)
11283 		return driver->name;
11284 	return empty;
11285 }
11286 
11287 static void __netdev_printk(const char *level, const struct net_device *dev,
11288 			    struct va_format *vaf)
11289 {
11290 	if (dev && dev->dev.parent) {
11291 		dev_printk_emit(level[1] - '0',
11292 				dev->dev.parent,
11293 				"%s %s %s%s: %pV",
11294 				dev_driver_string(dev->dev.parent),
11295 				dev_name(dev->dev.parent),
11296 				netdev_name(dev), netdev_reg_state(dev),
11297 				vaf);
11298 	} else if (dev) {
11299 		printk("%s%s%s: %pV",
11300 		       level, netdev_name(dev), netdev_reg_state(dev), vaf);
11301 	} else {
11302 		printk("%s(NULL net_device): %pV", level, vaf);
11303 	}
11304 }
11305 
11306 void netdev_printk(const char *level, const struct net_device *dev,
11307 		   const char *format, ...)
11308 {
11309 	struct va_format vaf;
11310 	va_list args;
11311 
11312 	va_start(args, format);
11313 
11314 	vaf.fmt = format;
11315 	vaf.va = &args;
11316 
11317 	__netdev_printk(level, dev, &vaf);
11318 
11319 	va_end(args);
11320 }
11321 EXPORT_SYMBOL(netdev_printk);
11322 
11323 #define define_netdev_printk_level(func, level)			\
11324 void func(const struct net_device *dev, const char *fmt, ...)	\
11325 {								\
11326 	struct va_format vaf;					\
11327 	va_list args;						\
11328 								\
11329 	va_start(args, fmt);					\
11330 								\
11331 	vaf.fmt = fmt;						\
11332 	vaf.va = &args;						\
11333 								\
11334 	__netdev_printk(level, dev, &vaf);			\
11335 								\
11336 	va_end(args);						\
11337 }								\
11338 EXPORT_SYMBOL(func);
11339 
11340 define_netdev_printk_level(netdev_emerg, KERN_EMERG);
11341 define_netdev_printk_level(netdev_alert, KERN_ALERT);
11342 define_netdev_printk_level(netdev_crit, KERN_CRIT);
11343 define_netdev_printk_level(netdev_err, KERN_ERR);
11344 define_netdev_printk_level(netdev_warn, KERN_WARNING);
11345 define_netdev_printk_level(netdev_notice, KERN_NOTICE);
11346 define_netdev_printk_level(netdev_info, KERN_INFO);
11347 
11348 static void __net_exit netdev_exit(struct net *net)
11349 {
11350 	kfree(net->dev_name_head);
11351 	kfree(net->dev_index_head);
11352 	if (net != &init_net)
11353 		WARN_ON_ONCE(!list_empty(&net->dev_base_head));
11354 }
11355 
11356 static struct pernet_operations __net_initdata netdev_net_ops = {
11357 	.init = netdev_init,
11358 	.exit = netdev_exit,
11359 };
11360 
11361 static void __net_exit default_device_exit(struct net *net)
11362 {
11363 	struct net_device *dev, *aux;
11364 	/*
11365 	 * Push all migratable network devices back to the
11366 	 * initial network namespace
11367 	 */
11368 	rtnl_lock();
11369 	for_each_netdev_safe(net, dev, aux) {
11370 		int err;
11371 		char fb_name[IFNAMSIZ];
11372 
11373 		/* Ignore unmoveable devices (i.e. loopback) */
11374 		if (dev->features & NETIF_F_NETNS_LOCAL)
11375 			continue;
11376 
11377 		/* Leave virtual devices for the generic cleanup */
11378 		if (dev->rtnl_link_ops && !dev->rtnl_link_ops->netns_refund)
11379 			continue;
11380 
11381 		/* Push remaining network devices to init_net */
11382 		snprintf(fb_name, IFNAMSIZ, "dev%d", dev->ifindex);
11383 		if (__dev_get_by_name(&init_net, fb_name))
11384 			snprintf(fb_name, IFNAMSIZ, "dev%%d");
11385 		err = dev_change_net_namespace(dev, &init_net, fb_name);
11386 		if (err) {
11387 			pr_emerg("%s: failed to move %s to init_net: %d\n",
11388 				 __func__, dev->name, err);
11389 			BUG();
11390 		}
11391 	}
11392 	rtnl_unlock();
11393 }
11394 
11395 static void __net_exit rtnl_lock_unregistering(struct list_head *net_list)
11396 {
11397 	/* Return with the rtnl_lock held when there are no network
11398 	 * devices unregistering in any network namespace in net_list.
11399 	 */
11400 	struct net *net;
11401 	bool unregistering;
11402 	DEFINE_WAIT_FUNC(wait, woken_wake_function);
11403 
11404 	add_wait_queue(&netdev_unregistering_wq, &wait);
11405 	for (;;) {
11406 		unregistering = false;
11407 		rtnl_lock();
11408 		list_for_each_entry(net, net_list, exit_list) {
11409 			if (net->dev_unreg_count > 0) {
11410 				unregistering = true;
11411 				break;
11412 			}
11413 		}
11414 		if (!unregistering)
11415 			break;
11416 		__rtnl_unlock();
11417 
11418 		wait_woken(&wait, TASK_UNINTERRUPTIBLE, MAX_SCHEDULE_TIMEOUT);
11419 	}
11420 	remove_wait_queue(&netdev_unregistering_wq, &wait);
11421 }
11422 
11423 static void __net_exit default_device_exit_batch(struct list_head *net_list)
11424 {
11425 	/* At exit all network devices most be removed from a network
11426 	 * namespace.  Do this in the reverse order of registration.
11427 	 * Do this across as many network namespaces as possible to
11428 	 * improve batching efficiency.
11429 	 */
11430 	struct net_device *dev;
11431 	struct net *net;
11432 	LIST_HEAD(dev_kill_list);
11433 
11434 	/* To prevent network device cleanup code from dereferencing
11435 	 * loopback devices or network devices that have been freed
11436 	 * wait here for all pending unregistrations to complete,
11437 	 * before unregistring the loopback device and allowing the
11438 	 * network namespace be freed.
11439 	 *
11440 	 * The netdev todo list containing all network devices
11441 	 * unregistrations that happen in default_device_exit_batch
11442 	 * will run in the rtnl_unlock() at the end of
11443 	 * default_device_exit_batch.
11444 	 */
11445 	rtnl_lock_unregistering(net_list);
11446 	list_for_each_entry(net, net_list, exit_list) {
11447 		for_each_netdev_reverse(net, dev) {
11448 			if (dev->rtnl_link_ops && dev->rtnl_link_ops->dellink)
11449 				dev->rtnl_link_ops->dellink(dev, &dev_kill_list);
11450 			else
11451 				unregister_netdevice_queue(dev, &dev_kill_list);
11452 		}
11453 	}
11454 	unregister_netdevice_many(&dev_kill_list);
11455 	rtnl_unlock();
11456 }
11457 
11458 static struct pernet_operations __net_initdata default_device_ops = {
11459 	.exit = default_device_exit,
11460 	.exit_batch = default_device_exit_batch,
11461 };
11462 
11463 /*
11464  *	Initialize the DEV module. At boot time this walks the device list and
11465  *	unhooks any devices that fail to initialise (normally hardware not
11466  *	present) and leaves us with a valid list of present and active devices.
11467  *
11468  */
11469 
11470 /*
11471  *       This is called single threaded during boot, so no need
11472  *       to take the rtnl semaphore.
11473  */
11474 static int __init net_dev_init(void)
11475 {
11476 	int i, rc = -ENOMEM;
11477 
11478 	BUG_ON(!dev_boot_phase);
11479 
11480 	if (dev_proc_init())
11481 		goto out;
11482 
11483 	if (netdev_kobject_init())
11484 		goto out;
11485 
11486 	INIT_LIST_HEAD(&ptype_all);
11487 	for (i = 0; i < PTYPE_HASH_SIZE; i++)
11488 		INIT_LIST_HEAD(&ptype_base[i]);
11489 
11490 	INIT_LIST_HEAD(&offload_base);
11491 
11492 	if (register_pernet_subsys(&netdev_net_ops))
11493 		goto out;
11494 
11495 	/*
11496 	 *	Initialise the packet receive queues.
11497 	 */
11498 
11499 	for_each_possible_cpu(i) {
11500 		struct work_struct *flush = per_cpu_ptr(&flush_works, i);
11501 		struct softnet_data *sd = &per_cpu(softnet_data, i);
11502 
11503 		INIT_WORK(flush, flush_backlog);
11504 
11505 		skb_queue_head_init(&sd->input_pkt_queue);
11506 		skb_queue_head_init(&sd->process_queue);
11507 #ifdef CONFIG_XFRM_OFFLOAD
11508 		skb_queue_head_init(&sd->xfrm_backlog);
11509 #endif
11510 		INIT_LIST_HEAD(&sd->poll_list);
11511 		sd->output_queue_tailp = &sd->output_queue;
11512 #ifdef CONFIG_RPS
11513 		INIT_CSD(&sd->csd, rps_trigger_softirq, sd);
11514 		sd->cpu = i;
11515 #endif
11516 
11517 		init_gro_hash(&sd->backlog);
11518 		sd->backlog.poll = process_backlog;
11519 		sd->backlog.weight = weight_p;
11520 	}
11521 
11522 	dev_boot_phase = 0;
11523 
11524 	/* The loopback device is special if any other network devices
11525 	 * is present in a network namespace the loopback device must
11526 	 * be present. Since we now dynamically allocate and free the
11527 	 * loopback device ensure this invariant is maintained by
11528 	 * keeping the loopback device as the first device on the
11529 	 * list of network devices.  Ensuring the loopback devices
11530 	 * is the first device that appears and the last network device
11531 	 * that disappears.
11532 	 */
11533 	if (register_pernet_device(&loopback_net_ops))
11534 		goto out;
11535 
11536 	if (register_pernet_device(&default_device_ops))
11537 		goto out;
11538 
11539 	open_softirq(NET_TX_SOFTIRQ, net_tx_action);
11540 	open_softirq(NET_RX_SOFTIRQ, net_rx_action);
11541 
11542 	rc = cpuhp_setup_state_nocalls(CPUHP_NET_DEV_DEAD, "net/dev:dead",
11543 				       NULL, dev_cpu_dead);
11544 	WARN_ON(rc < 0);
11545 	rc = 0;
11546 out:
11547 	return rc;
11548 }
11549 
11550 subsys_initcall(net_dev_init);
11551