xref: /openbmc/linux/include/linux/netdevice.h (revision 3d3337de)
1 /*
2  * INET		An implementation of the TCP/IP protocol suite for the LINUX
3  *		operating system.  INET is implemented using the  BSD Socket
4  *		interface as the means of communication with the user level.
5  *
6  *		Definitions for the Interfaces handler.
7  *
8  * Version:	@(#)dev.h	1.0.10	08/12/93
9  *
10  * Authors:	Ross Biro
11  *		Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
12  *		Corey Minyard <wf-rch!minyard@relay.EU.net>
13  *		Donald J. Becker, <becker@cesdis.gsfc.nasa.gov>
14  *		Alan Cox, <alan@lxorguk.ukuu.org.uk>
15  *		Bjorn Ekwall. <bj0rn@blox.se>
16  *              Pekka Riikonen <priikone@poseidon.pspt.fi>
17  *
18  *		This program is free software; you can redistribute it and/or
19  *		modify it under the terms of the GNU General Public License
20  *		as published by the Free Software Foundation; either version
21  *		2 of the License, or (at your option) any later version.
22  *
23  *		Moved to /usr/include/linux for NET3
24  */
25 #ifndef _LINUX_NETDEVICE_H
26 #define _LINUX_NETDEVICE_H
27 
28 #include <linux/pm_qos.h>
29 #include <linux/timer.h>
30 #include <linux/bug.h>
31 #include <linux/delay.h>
32 #include <linux/atomic.h>
33 #include <linux/prefetch.h>
34 #include <asm/cache.h>
35 #include <asm/byteorder.h>
36 
37 #include <linux/percpu.h>
38 #include <linux/rculist.h>
39 #include <linux/dmaengine.h>
40 #include <linux/workqueue.h>
41 #include <linux/dynamic_queue_limits.h>
42 
43 #include <linux/ethtool.h>
44 #include <net/net_namespace.h>
45 #include <net/dsa.h>
46 #ifdef CONFIG_DCB
47 #include <net/dcbnl.h>
48 #endif
49 #include <net/netprio_cgroup.h>
50 
51 #include <linux/netdev_features.h>
52 #include <linux/neighbour.h>
53 #include <uapi/linux/netdevice.h>
54 #include <uapi/linux/if_bonding.h>
55 
56 struct netpoll_info;
57 struct device;
58 struct phy_device;
59 /* 802.11 specific */
60 struct wireless_dev;
61 /* 802.15.4 specific */
62 struct wpan_dev;
63 struct mpls_dev;
64 
65 void netdev_set_default_ethtool_ops(struct net_device *dev,
66 				    const struct ethtool_ops *ops);
67 
68 /* Backlog congestion levels */
69 #define NET_RX_SUCCESS		0	/* keep 'em coming, baby */
70 #define NET_RX_DROP		1	/* packet dropped */
71 
72 /*
73  * Transmit return codes: transmit return codes originate from three different
74  * namespaces:
75  *
76  * - qdisc return codes
77  * - driver transmit return codes
78  * - errno values
79  *
80  * Drivers are allowed to return any one of those in their hard_start_xmit()
81  * function. Real network devices commonly used with qdiscs should only return
82  * the driver transmit return codes though - when qdiscs are used, the actual
83  * transmission happens asynchronously, so the value is not propagated to
84  * higher layers. Virtual network devices transmit synchronously, in this case
85  * the driver transmit return codes are consumed by dev_queue_xmit(), all
86  * others are propagated to higher layers.
87  */
88 
89 /* qdisc ->enqueue() return codes. */
90 #define NET_XMIT_SUCCESS	0x00
91 #define NET_XMIT_DROP		0x01	/* skb dropped			*/
92 #define NET_XMIT_CN		0x02	/* congestion notification	*/
93 #define NET_XMIT_POLICED	0x03	/* skb is shot by police	*/
94 #define NET_XMIT_MASK		0x0f	/* qdisc flags in net/sch_generic.h */
95 
96 /* NET_XMIT_CN is special. It does not guarantee that this packet is lost. It
97  * indicates that the device will soon be dropping packets, or already drops
98  * some packets of the same priority; prompting us to send less aggressively. */
99 #define net_xmit_eval(e)	((e) == NET_XMIT_CN ? 0 : (e))
100 #define net_xmit_errno(e)	((e) != NET_XMIT_CN ? -ENOBUFS : 0)
101 
102 /* Driver transmit return codes */
103 #define NETDEV_TX_MASK		0xf0
104 
105 enum netdev_tx {
106 	__NETDEV_TX_MIN	 = INT_MIN,	/* make sure enum is signed */
107 	NETDEV_TX_OK	 = 0x00,	/* driver took care of packet */
108 	NETDEV_TX_BUSY	 = 0x10,	/* driver tx path was busy*/
109 	NETDEV_TX_LOCKED = 0x20,	/* driver tx lock was already taken */
110 };
111 typedef enum netdev_tx netdev_tx_t;
112 
113 /*
114  * Current order: NETDEV_TX_MASK > NET_XMIT_MASK >= 0 is significant;
115  * hard_start_xmit() return < NET_XMIT_MASK means skb was consumed.
116  */
117 static inline bool dev_xmit_complete(int rc)
118 {
119 	/*
120 	 * Positive cases with an skb consumed by a driver:
121 	 * - successful transmission (rc == NETDEV_TX_OK)
122 	 * - error while transmitting (rc < 0)
123 	 * - error while queueing to a different device (rc & NET_XMIT_MASK)
124 	 */
125 	if (likely(rc < NET_XMIT_MASK))
126 		return true;
127 
128 	return false;
129 }
130 
131 /*
132  *	Compute the worst case header length according to the protocols
133  *	used.
134  */
135 
136 #if defined(CONFIG_WLAN) || IS_ENABLED(CONFIG_AX25)
137 # if defined(CONFIG_MAC80211_MESH)
138 #  define LL_MAX_HEADER 128
139 # else
140 #  define LL_MAX_HEADER 96
141 # endif
142 #else
143 # define LL_MAX_HEADER 32
144 #endif
145 
146 #if !IS_ENABLED(CONFIG_NET_IPIP) && !IS_ENABLED(CONFIG_NET_IPGRE) && \
147     !IS_ENABLED(CONFIG_IPV6_SIT) && !IS_ENABLED(CONFIG_IPV6_TUNNEL)
148 #define MAX_HEADER LL_MAX_HEADER
149 #else
150 #define MAX_HEADER (LL_MAX_HEADER + 48)
151 #endif
152 
153 /*
154  *	Old network device statistics. Fields are native words
155  *	(unsigned long) so they can be read and written atomically.
156  */
157 
158 struct net_device_stats {
159 	unsigned long	rx_packets;
160 	unsigned long	tx_packets;
161 	unsigned long	rx_bytes;
162 	unsigned long	tx_bytes;
163 	unsigned long	rx_errors;
164 	unsigned long	tx_errors;
165 	unsigned long	rx_dropped;
166 	unsigned long	tx_dropped;
167 	unsigned long	multicast;
168 	unsigned long	collisions;
169 	unsigned long	rx_length_errors;
170 	unsigned long	rx_over_errors;
171 	unsigned long	rx_crc_errors;
172 	unsigned long	rx_frame_errors;
173 	unsigned long	rx_fifo_errors;
174 	unsigned long	rx_missed_errors;
175 	unsigned long	tx_aborted_errors;
176 	unsigned long	tx_carrier_errors;
177 	unsigned long	tx_fifo_errors;
178 	unsigned long	tx_heartbeat_errors;
179 	unsigned long	tx_window_errors;
180 	unsigned long	rx_compressed;
181 	unsigned long	tx_compressed;
182 };
183 
184 
185 #include <linux/cache.h>
186 #include <linux/skbuff.h>
187 
188 #ifdef CONFIG_RPS
189 #include <linux/static_key.h>
190 extern struct static_key rps_needed;
191 #endif
192 
193 struct neighbour;
194 struct neigh_parms;
195 struct sk_buff;
196 
197 struct netdev_hw_addr {
198 	struct list_head	list;
199 	unsigned char		addr[MAX_ADDR_LEN];
200 	unsigned char		type;
201 #define NETDEV_HW_ADDR_T_LAN		1
202 #define NETDEV_HW_ADDR_T_SAN		2
203 #define NETDEV_HW_ADDR_T_SLAVE		3
204 #define NETDEV_HW_ADDR_T_UNICAST	4
205 #define NETDEV_HW_ADDR_T_MULTICAST	5
206 	bool			global_use;
207 	int			sync_cnt;
208 	int			refcount;
209 	int			synced;
210 	struct rcu_head		rcu_head;
211 };
212 
213 struct netdev_hw_addr_list {
214 	struct list_head	list;
215 	int			count;
216 };
217 
218 #define netdev_hw_addr_list_count(l) ((l)->count)
219 #define netdev_hw_addr_list_empty(l) (netdev_hw_addr_list_count(l) == 0)
220 #define netdev_hw_addr_list_for_each(ha, l) \
221 	list_for_each_entry(ha, &(l)->list, list)
222 
223 #define netdev_uc_count(dev) netdev_hw_addr_list_count(&(dev)->uc)
224 #define netdev_uc_empty(dev) netdev_hw_addr_list_empty(&(dev)->uc)
225 #define netdev_for_each_uc_addr(ha, dev) \
226 	netdev_hw_addr_list_for_each(ha, &(dev)->uc)
227 
228 #define netdev_mc_count(dev) netdev_hw_addr_list_count(&(dev)->mc)
229 #define netdev_mc_empty(dev) netdev_hw_addr_list_empty(&(dev)->mc)
230 #define netdev_for_each_mc_addr(ha, dev) \
231 	netdev_hw_addr_list_for_each(ha, &(dev)->mc)
232 
233 struct hh_cache {
234 	u16		hh_len;
235 	u16		__pad;
236 	seqlock_t	hh_lock;
237 
238 	/* cached hardware header; allow for machine alignment needs.        */
239 #define HH_DATA_MOD	16
240 #define HH_DATA_OFF(__len) \
241 	(HH_DATA_MOD - (((__len - 1) & (HH_DATA_MOD - 1)) + 1))
242 #define HH_DATA_ALIGN(__len) \
243 	(((__len)+(HH_DATA_MOD-1))&~(HH_DATA_MOD - 1))
244 	unsigned long	hh_data[HH_DATA_ALIGN(LL_MAX_HEADER) / sizeof(long)];
245 };
246 
247 /* Reserve HH_DATA_MOD byte aligned hard_header_len, but at least that much.
248  * Alternative is:
249  *   dev->hard_header_len ? (dev->hard_header_len +
250  *                           (HH_DATA_MOD - 1)) & ~(HH_DATA_MOD - 1) : 0
251  *
252  * We could use other alignment values, but we must maintain the
253  * relationship HH alignment <= LL alignment.
254  */
255 #define LL_RESERVED_SPACE(dev) \
256 	((((dev)->hard_header_len+(dev)->needed_headroom)&~(HH_DATA_MOD - 1)) + HH_DATA_MOD)
257 #define LL_RESERVED_SPACE_EXTRA(dev,extra) \
258 	((((dev)->hard_header_len+(dev)->needed_headroom+(extra))&~(HH_DATA_MOD - 1)) + HH_DATA_MOD)
259 
260 struct header_ops {
261 	int	(*create) (struct sk_buff *skb, struct net_device *dev,
262 			   unsigned short type, const void *daddr,
263 			   const void *saddr, unsigned int len);
264 	int	(*parse)(const struct sk_buff *skb, unsigned char *haddr);
265 	int	(*cache)(const struct neighbour *neigh, struct hh_cache *hh, __be16 type);
266 	void	(*cache_update)(struct hh_cache *hh,
267 				const struct net_device *dev,
268 				const unsigned char *haddr);
269 };
270 
271 /* These flag bits are private to the generic network queueing
272  * layer, they may not be explicitly referenced by any other
273  * code.
274  */
275 
276 enum netdev_state_t {
277 	__LINK_STATE_START,
278 	__LINK_STATE_PRESENT,
279 	__LINK_STATE_NOCARRIER,
280 	__LINK_STATE_LINKWATCH_PENDING,
281 	__LINK_STATE_DORMANT,
282 };
283 
284 
285 /*
286  * This structure holds at boot time configured netdevice settings. They
287  * are then used in the device probing.
288  */
289 struct netdev_boot_setup {
290 	char name[IFNAMSIZ];
291 	struct ifmap map;
292 };
293 #define NETDEV_BOOT_SETUP_MAX 8
294 
295 int __init netdev_boot_setup(char *str);
296 
297 /*
298  * Structure for NAPI scheduling similar to tasklet but with weighting
299  */
300 struct napi_struct {
301 	/* The poll_list must only be managed by the entity which
302 	 * changes the state of the NAPI_STATE_SCHED bit.  This means
303 	 * whoever atomically sets that bit can add this napi_struct
304 	 * to the per-cpu poll_list, and whoever clears that bit
305 	 * can remove from the list right before clearing the bit.
306 	 */
307 	struct list_head	poll_list;
308 
309 	unsigned long		state;
310 	int			weight;
311 	unsigned int		gro_count;
312 	int			(*poll)(struct napi_struct *, int);
313 #ifdef CONFIG_NETPOLL
314 	spinlock_t		poll_lock;
315 	int			poll_owner;
316 #endif
317 	struct net_device	*dev;
318 	struct sk_buff		*gro_list;
319 	struct sk_buff		*skb;
320 	struct hrtimer		timer;
321 	struct list_head	dev_list;
322 	struct hlist_node	napi_hash_node;
323 	unsigned int		napi_id;
324 };
325 
326 enum {
327 	NAPI_STATE_SCHED,	/* Poll is scheduled */
328 	NAPI_STATE_DISABLE,	/* Disable pending */
329 	NAPI_STATE_NPSVC,	/* Netpoll - don't dequeue from poll_list */
330 	NAPI_STATE_HASHED,	/* In NAPI hash */
331 };
332 
333 enum gro_result {
334 	GRO_MERGED,
335 	GRO_MERGED_FREE,
336 	GRO_HELD,
337 	GRO_NORMAL,
338 	GRO_DROP,
339 };
340 typedef enum gro_result gro_result_t;
341 
342 /*
343  * enum rx_handler_result - Possible return values for rx_handlers.
344  * @RX_HANDLER_CONSUMED: skb was consumed by rx_handler, do not process it
345  * further.
346  * @RX_HANDLER_ANOTHER: Do another round in receive path. This is indicated in
347  * case skb->dev was changed by rx_handler.
348  * @RX_HANDLER_EXACT: Force exact delivery, no wildcard.
349  * @RX_HANDLER_PASS: Do nothing, passe the skb as if no rx_handler was called.
350  *
351  * rx_handlers are functions called from inside __netif_receive_skb(), to do
352  * special processing of the skb, prior to delivery to protocol handlers.
353  *
354  * Currently, a net_device can only have a single rx_handler registered. Trying
355  * to register a second rx_handler will return -EBUSY.
356  *
357  * To register a rx_handler on a net_device, use netdev_rx_handler_register().
358  * To unregister a rx_handler on a net_device, use
359  * netdev_rx_handler_unregister().
360  *
361  * Upon return, rx_handler is expected to tell __netif_receive_skb() what to
362  * do with the skb.
363  *
364  * If the rx_handler consumed to skb in some way, it should return
365  * RX_HANDLER_CONSUMED. This is appropriate when the rx_handler arranged for
366  * the skb to be delivered in some other ways.
367  *
368  * If the rx_handler changed skb->dev, to divert the skb to another
369  * net_device, it should return RX_HANDLER_ANOTHER. The rx_handler for the
370  * new device will be called if it exists.
371  *
372  * If the rx_handler consider the skb should be ignored, it should return
373  * RX_HANDLER_EXACT. The skb will only be delivered to protocol handlers that
374  * are registered on exact device (ptype->dev == skb->dev).
375  *
376  * If the rx_handler didn't changed skb->dev, but want the skb to be normally
377  * delivered, it should return RX_HANDLER_PASS.
378  *
379  * A device without a registered rx_handler will behave as if rx_handler
380  * returned RX_HANDLER_PASS.
381  */
382 
383 enum rx_handler_result {
384 	RX_HANDLER_CONSUMED,
385 	RX_HANDLER_ANOTHER,
386 	RX_HANDLER_EXACT,
387 	RX_HANDLER_PASS,
388 };
389 typedef enum rx_handler_result rx_handler_result_t;
390 typedef rx_handler_result_t rx_handler_func_t(struct sk_buff **pskb);
391 
392 void __napi_schedule(struct napi_struct *n);
393 void __napi_schedule_irqoff(struct napi_struct *n);
394 
395 static inline bool napi_disable_pending(struct napi_struct *n)
396 {
397 	return test_bit(NAPI_STATE_DISABLE, &n->state);
398 }
399 
400 /**
401  *	napi_schedule_prep - check if napi can be scheduled
402  *	@n: napi context
403  *
404  * Test if NAPI routine is already running, and if not mark
405  * it as running.  This is used as a condition variable
406  * insure only one NAPI poll instance runs.  We also make
407  * sure there is no pending NAPI disable.
408  */
409 static inline bool napi_schedule_prep(struct napi_struct *n)
410 {
411 	return !napi_disable_pending(n) &&
412 		!test_and_set_bit(NAPI_STATE_SCHED, &n->state);
413 }
414 
415 /**
416  *	napi_schedule - schedule NAPI poll
417  *	@n: napi context
418  *
419  * Schedule NAPI poll routine to be called if it is not already
420  * running.
421  */
422 static inline void napi_schedule(struct napi_struct *n)
423 {
424 	if (napi_schedule_prep(n))
425 		__napi_schedule(n);
426 }
427 
428 /**
429  *	napi_schedule_irqoff - schedule NAPI poll
430  *	@n: napi context
431  *
432  * Variant of napi_schedule(), assuming hard irqs are masked.
433  */
434 static inline void napi_schedule_irqoff(struct napi_struct *n)
435 {
436 	if (napi_schedule_prep(n))
437 		__napi_schedule_irqoff(n);
438 }
439 
440 /* Try to reschedule poll. Called by dev->poll() after napi_complete().  */
441 static inline bool napi_reschedule(struct napi_struct *napi)
442 {
443 	if (napi_schedule_prep(napi)) {
444 		__napi_schedule(napi);
445 		return true;
446 	}
447 	return false;
448 }
449 
450 void __napi_complete(struct napi_struct *n);
451 void napi_complete_done(struct napi_struct *n, int work_done);
452 /**
453  *	napi_complete - NAPI processing complete
454  *	@n: napi context
455  *
456  * Mark NAPI processing as complete.
457  * Consider using napi_complete_done() instead.
458  */
459 static inline void napi_complete(struct napi_struct *n)
460 {
461 	return napi_complete_done(n, 0);
462 }
463 
464 /**
465  *	napi_by_id - lookup a NAPI by napi_id
466  *	@napi_id: hashed napi_id
467  *
468  * lookup @napi_id in napi_hash table
469  * must be called under rcu_read_lock()
470  */
471 struct napi_struct *napi_by_id(unsigned int napi_id);
472 
473 /**
474  *	napi_hash_add - add a NAPI to global hashtable
475  *	@napi: napi context
476  *
477  * generate a new napi_id and store a @napi under it in napi_hash
478  */
479 void napi_hash_add(struct napi_struct *napi);
480 
481 /**
482  *	napi_hash_del - remove a NAPI from global table
483  *	@napi: napi context
484  *
485  * Warning: caller must observe rcu grace period
486  * before freeing memory containing @napi
487  */
488 void napi_hash_del(struct napi_struct *napi);
489 
490 /**
491  *	napi_disable - prevent NAPI from scheduling
492  *	@n: napi context
493  *
494  * Stop NAPI from being scheduled on this context.
495  * Waits till any outstanding processing completes.
496  */
497 void napi_disable(struct napi_struct *n);
498 
499 /**
500  *	napi_enable - enable NAPI scheduling
501  *	@n: napi context
502  *
503  * Resume NAPI from being scheduled on this context.
504  * Must be paired with napi_disable.
505  */
506 static inline void napi_enable(struct napi_struct *n)
507 {
508 	BUG_ON(!test_bit(NAPI_STATE_SCHED, &n->state));
509 	smp_mb__before_atomic();
510 	clear_bit(NAPI_STATE_SCHED, &n->state);
511 }
512 
513 #ifdef CONFIG_SMP
514 /**
515  *	napi_synchronize - wait until NAPI is not running
516  *	@n: napi context
517  *
518  * Wait until NAPI is done being scheduled on this context.
519  * Waits till any outstanding processing completes but
520  * does not disable future activations.
521  */
522 static inline void napi_synchronize(const struct napi_struct *n)
523 {
524 	while (test_bit(NAPI_STATE_SCHED, &n->state))
525 		msleep(1);
526 }
527 #else
528 # define napi_synchronize(n)	barrier()
529 #endif
530 
531 enum netdev_queue_state_t {
532 	__QUEUE_STATE_DRV_XOFF,
533 	__QUEUE_STATE_STACK_XOFF,
534 	__QUEUE_STATE_FROZEN,
535 };
536 
537 #define QUEUE_STATE_DRV_XOFF	(1 << __QUEUE_STATE_DRV_XOFF)
538 #define QUEUE_STATE_STACK_XOFF	(1 << __QUEUE_STATE_STACK_XOFF)
539 #define QUEUE_STATE_FROZEN	(1 << __QUEUE_STATE_FROZEN)
540 
541 #define QUEUE_STATE_ANY_XOFF	(QUEUE_STATE_DRV_XOFF | QUEUE_STATE_STACK_XOFF)
542 #define QUEUE_STATE_ANY_XOFF_OR_FROZEN (QUEUE_STATE_ANY_XOFF | \
543 					QUEUE_STATE_FROZEN)
544 #define QUEUE_STATE_DRV_XOFF_OR_FROZEN (QUEUE_STATE_DRV_XOFF | \
545 					QUEUE_STATE_FROZEN)
546 
547 /*
548  * __QUEUE_STATE_DRV_XOFF is used by drivers to stop the transmit queue.  The
549  * netif_tx_* functions below are used to manipulate this flag.  The
550  * __QUEUE_STATE_STACK_XOFF flag is used by the stack to stop the transmit
551  * queue independently.  The netif_xmit_*stopped functions below are called
552  * to check if the queue has been stopped by the driver or stack (either
553  * of the XOFF bits are set in the state).  Drivers should not need to call
554  * netif_xmit*stopped functions, they should only be using netif_tx_*.
555  */
556 
557 struct netdev_queue {
558 /*
559  * read mostly part
560  */
561 	struct net_device	*dev;
562 	struct Qdisc __rcu	*qdisc;
563 	struct Qdisc		*qdisc_sleeping;
564 #ifdef CONFIG_SYSFS
565 	struct kobject		kobj;
566 #endif
567 #if defined(CONFIG_XPS) && defined(CONFIG_NUMA)
568 	int			numa_node;
569 #endif
570 /*
571  * write mostly part
572  */
573 	spinlock_t		_xmit_lock ____cacheline_aligned_in_smp;
574 	int			xmit_lock_owner;
575 	/*
576 	 * please use this field instead of dev->trans_start
577 	 */
578 	unsigned long		trans_start;
579 
580 	/*
581 	 * Number of TX timeouts for this queue
582 	 * (/sys/class/net/DEV/Q/trans_timeout)
583 	 */
584 	unsigned long		trans_timeout;
585 
586 	unsigned long		state;
587 
588 #ifdef CONFIG_BQL
589 	struct dql		dql;
590 #endif
591 	unsigned long		tx_maxrate;
592 } ____cacheline_aligned_in_smp;
593 
594 static inline int netdev_queue_numa_node_read(const struct netdev_queue *q)
595 {
596 #if defined(CONFIG_XPS) && defined(CONFIG_NUMA)
597 	return q->numa_node;
598 #else
599 	return NUMA_NO_NODE;
600 #endif
601 }
602 
603 static inline void netdev_queue_numa_node_write(struct netdev_queue *q, int node)
604 {
605 #if defined(CONFIG_XPS) && defined(CONFIG_NUMA)
606 	q->numa_node = node;
607 #endif
608 }
609 
610 #ifdef CONFIG_RPS
611 /*
612  * This structure holds an RPS map which can be of variable length.  The
613  * map is an array of CPUs.
614  */
615 struct rps_map {
616 	unsigned int len;
617 	struct rcu_head rcu;
618 	u16 cpus[0];
619 };
620 #define RPS_MAP_SIZE(_num) (sizeof(struct rps_map) + ((_num) * sizeof(u16)))
621 
622 /*
623  * The rps_dev_flow structure contains the mapping of a flow to a CPU, the
624  * tail pointer for that CPU's input queue at the time of last enqueue, and
625  * a hardware filter index.
626  */
627 struct rps_dev_flow {
628 	u16 cpu;
629 	u16 filter;
630 	unsigned int last_qtail;
631 };
632 #define RPS_NO_FILTER 0xffff
633 
634 /*
635  * The rps_dev_flow_table structure contains a table of flow mappings.
636  */
637 struct rps_dev_flow_table {
638 	unsigned int mask;
639 	struct rcu_head rcu;
640 	struct rps_dev_flow flows[0];
641 };
642 #define RPS_DEV_FLOW_TABLE_SIZE(_num) (sizeof(struct rps_dev_flow_table) + \
643     ((_num) * sizeof(struct rps_dev_flow)))
644 
645 /*
646  * The rps_sock_flow_table contains mappings of flows to the last CPU
647  * on which they were processed by the application (set in recvmsg).
648  * Each entry is a 32bit value. Upper part is the high order bits
649  * of flow hash, lower part is cpu number.
650  * rps_cpu_mask is used to partition the space, depending on number of
651  * possible cpus : rps_cpu_mask = roundup_pow_of_two(nr_cpu_ids) - 1
652  * For example, if 64 cpus are possible, rps_cpu_mask = 0x3f,
653  * meaning we use 32-6=26 bits for the hash.
654  */
655 struct rps_sock_flow_table {
656 	u32	mask;
657 
658 	u32	ents[0] ____cacheline_aligned_in_smp;
659 };
660 #define	RPS_SOCK_FLOW_TABLE_SIZE(_num) (offsetof(struct rps_sock_flow_table, ents[_num]))
661 
662 #define RPS_NO_CPU 0xffff
663 
664 extern u32 rps_cpu_mask;
665 extern struct rps_sock_flow_table __rcu *rps_sock_flow_table;
666 
667 static inline void rps_record_sock_flow(struct rps_sock_flow_table *table,
668 					u32 hash)
669 {
670 	if (table && hash) {
671 		unsigned int index = hash & table->mask;
672 		u32 val = hash & ~rps_cpu_mask;
673 
674 		/* We only give a hint, preemption can change cpu under us */
675 		val |= raw_smp_processor_id();
676 
677 		if (table->ents[index] != val)
678 			table->ents[index] = val;
679 	}
680 }
681 
682 #ifdef CONFIG_RFS_ACCEL
683 bool rps_may_expire_flow(struct net_device *dev, u16 rxq_index, u32 flow_id,
684 			 u16 filter_id);
685 #endif
686 #endif /* CONFIG_RPS */
687 
688 /* This structure contains an instance of an RX queue. */
689 struct netdev_rx_queue {
690 #ifdef CONFIG_RPS
691 	struct rps_map __rcu		*rps_map;
692 	struct rps_dev_flow_table __rcu	*rps_flow_table;
693 #endif
694 	struct kobject			kobj;
695 	struct net_device		*dev;
696 } ____cacheline_aligned_in_smp;
697 
698 /*
699  * RX queue sysfs structures and functions.
700  */
701 struct rx_queue_attribute {
702 	struct attribute attr;
703 	ssize_t (*show)(struct netdev_rx_queue *queue,
704 	    struct rx_queue_attribute *attr, char *buf);
705 	ssize_t (*store)(struct netdev_rx_queue *queue,
706 	    struct rx_queue_attribute *attr, const char *buf, size_t len);
707 };
708 
709 #ifdef CONFIG_XPS
710 /*
711  * This structure holds an XPS map which can be of variable length.  The
712  * map is an array of queues.
713  */
714 struct xps_map {
715 	unsigned int len;
716 	unsigned int alloc_len;
717 	struct rcu_head rcu;
718 	u16 queues[0];
719 };
720 #define XPS_MAP_SIZE(_num) (sizeof(struct xps_map) + ((_num) * sizeof(u16)))
721 #define XPS_MIN_MAP_ALLOC ((L1_CACHE_BYTES - sizeof(struct xps_map))	\
722     / sizeof(u16))
723 
724 /*
725  * This structure holds all XPS maps for device.  Maps are indexed by CPU.
726  */
727 struct xps_dev_maps {
728 	struct rcu_head rcu;
729 	struct xps_map __rcu *cpu_map[0];
730 };
731 #define XPS_DEV_MAPS_SIZE (sizeof(struct xps_dev_maps) +		\
732     (nr_cpu_ids * sizeof(struct xps_map *)))
733 #endif /* CONFIG_XPS */
734 
735 #define TC_MAX_QUEUE	16
736 #define TC_BITMASK	15
737 /* HW offloaded queuing disciplines txq count and offset maps */
738 struct netdev_tc_txq {
739 	u16 count;
740 	u16 offset;
741 };
742 
743 #if defined(CONFIG_FCOE) || defined(CONFIG_FCOE_MODULE)
744 /*
745  * This structure is to hold information about the device
746  * configured to run FCoE protocol stack.
747  */
748 struct netdev_fcoe_hbainfo {
749 	char	manufacturer[64];
750 	char	serial_number[64];
751 	char	hardware_version[64];
752 	char	driver_version[64];
753 	char	optionrom_version[64];
754 	char	firmware_version[64];
755 	char	model[256];
756 	char	model_description[256];
757 };
758 #endif
759 
760 #define MAX_PHYS_ITEM_ID_LEN 32
761 
762 /* This structure holds a unique identifier to identify some
763  * physical item (port for example) used by a netdevice.
764  */
765 struct netdev_phys_item_id {
766 	unsigned char id[MAX_PHYS_ITEM_ID_LEN];
767 	unsigned char id_len;
768 };
769 
770 typedef u16 (*select_queue_fallback_t)(struct net_device *dev,
771 				       struct sk_buff *skb);
772 
773 /*
774  * This structure defines the management hooks for network devices.
775  * The following hooks can be defined; unless noted otherwise, they are
776  * optional and can be filled with a null pointer.
777  *
778  * int (*ndo_init)(struct net_device *dev);
779  *     This function is called once when network device is registered.
780  *     The network device can use this to any late stage initializaton
781  *     or semantic validattion. It can fail with an error code which will
782  *     be propogated back to register_netdev
783  *
784  * void (*ndo_uninit)(struct net_device *dev);
785  *     This function is called when device is unregistered or when registration
786  *     fails. It is not called if init fails.
787  *
788  * int (*ndo_open)(struct net_device *dev);
789  *     This function is called when network device transistions to the up
790  *     state.
791  *
792  * int (*ndo_stop)(struct net_device *dev);
793  *     This function is called when network device transistions to the down
794  *     state.
795  *
796  * netdev_tx_t (*ndo_start_xmit)(struct sk_buff *skb,
797  *                               struct net_device *dev);
798  *	Called when a packet needs to be transmitted.
799  *	Returns NETDEV_TX_OK.  Can return NETDEV_TX_BUSY, but you should stop
800  *	the queue before that can happen; it's for obsolete devices and weird
801  *	corner cases, but the stack really does a non-trivial amount
802  *	of useless work if you return NETDEV_TX_BUSY.
803  *        (can also return NETDEV_TX_LOCKED iff NETIF_F_LLTX)
804  *	Required can not be NULL.
805  *
806  * u16 (*ndo_select_queue)(struct net_device *dev, struct sk_buff *skb,
807  *                         void *accel_priv, select_queue_fallback_t fallback);
808  *	Called to decide which queue to when device supports multiple
809  *	transmit queues.
810  *
811  * void (*ndo_change_rx_flags)(struct net_device *dev, int flags);
812  *	This function is called to allow device receiver to make
813  *	changes to configuration when multicast or promiscious is enabled.
814  *
815  * void (*ndo_set_rx_mode)(struct net_device *dev);
816  *	This function is called device changes address list filtering.
817  *	If driver handles unicast address filtering, it should set
818  *	IFF_UNICAST_FLT to its priv_flags.
819  *
820  * int (*ndo_set_mac_address)(struct net_device *dev, void *addr);
821  *	This function  is called when the Media Access Control address
822  *	needs to be changed. If this interface is not defined, the
823  *	mac address can not be changed.
824  *
825  * int (*ndo_validate_addr)(struct net_device *dev);
826  *	Test if Media Access Control address is valid for the device.
827  *
828  * int (*ndo_do_ioctl)(struct net_device *dev, struct ifreq *ifr, int cmd);
829  *	Called when a user request an ioctl which can't be handled by
830  *	the generic interface code. If not defined ioctl's return
831  *	not supported error code.
832  *
833  * int (*ndo_set_config)(struct net_device *dev, struct ifmap *map);
834  *	Used to set network devices bus interface parameters. This interface
835  *	is retained for legacy reason, new devices should use the bus
836  *	interface (PCI) for low level management.
837  *
838  * int (*ndo_change_mtu)(struct net_device *dev, int new_mtu);
839  *	Called when a user wants to change the Maximum Transfer Unit
840  *	of a device. If not defined, any request to change MTU will
841  *	will return an error.
842  *
843  * void (*ndo_tx_timeout)(struct net_device *dev);
844  *	Callback uses when the transmitter has not made any progress
845  *	for dev->watchdog ticks.
846  *
847  * struct rtnl_link_stats64* (*ndo_get_stats64)(struct net_device *dev,
848  *                      struct rtnl_link_stats64 *storage);
849  * struct net_device_stats* (*ndo_get_stats)(struct net_device *dev);
850  *	Called when a user wants to get the network device usage
851  *	statistics. Drivers must do one of the following:
852  *	1. Define @ndo_get_stats64 to fill in a zero-initialised
853  *	   rtnl_link_stats64 structure passed by the caller.
854  *	2. Define @ndo_get_stats to update a net_device_stats structure
855  *	   (which should normally be dev->stats) and return a pointer to
856  *	   it. The structure may be changed asynchronously only if each
857  *	   field is written atomically.
858  *	3. Update dev->stats asynchronously and atomically, and define
859  *	   neither operation.
860  *
861  * int (*ndo_vlan_rx_add_vid)(struct net_device *dev, __be16 proto, u16 vid);
862  *	If device support VLAN filtering this function is called when a
863  *	VLAN id is registered.
864  *
865  * int (*ndo_vlan_rx_kill_vid)(struct net_device *dev, __be16 proto, u16 vid);
866  *	If device support VLAN filtering this function is called when a
867  *	VLAN id is unregistered.
868  *
869  * void (*ndo_poll_controller)(struct net_device *dev);
870  *
871  *	SR-IOV management functions.
872  * int (*ndo_set_vf_mac)(struct net_device *dev, int vf, u8* mac);
873  * int (*ndo_set_vf_vlan)(struct net_device *dev, int vf, u16 vlan, u8 qos);
874  * int (*ndo_set_vf_rate)(struct net_device *dev, int vf, int min_tx_rate,
875  *			  int max_tx_rate);
876  * int (*ndo_set_vf_spoofchk)(struct net_device *dev, int vf, bool setting);
877  * int (*ndo_get_vf_config)(struct net_device *dev,
878  *			    int vf, struct ifla_vf_info *ivf);
879  * int (*ndo_set_vf_link_state)(struct net_device *dev, int vf, int link_state);
880  * int (*ndo_set_vf_port)(struct net_device *dev, int vf,
881  *			  struct nlattr *port[]);
882  *
883  *      Enable or disable the VF ability to query its RSS Redirection Table and
884  *      Hash Key. This is needed since on some devices VF share this information
885  *      with PF and querying it may adduce a theoretical security risk.
886  * int (*ndo_set_vf_rss_query_en)(struct net_device *dev, int vf, bool setting);
887  * int (*ndo_get_vf_port)(struct net_device *dev, int vf, struct sk_buff *skb);
888  * int (*ndo_setup_tc)(struct net_device *dev, u8 tc)
889  * 	Called to setup 'tc' number of traffic classes in the net device. This
890  * 	is always called from the stack with the rtnl lock held and netif tx
891  * 	queues stopped. This allows the netdevice to perform queue management
892  * 	safely.
893  *
894  *	Fiber Channel over Ethernet (FCoE) offload functions.
895  * int (*ndo_fcoe_enable)(struct net_device *dev);
896  *	Called when the FCoE protocol stack wants to start using LLD for FCoE
897  *	so the underlying device can perform whatever needed configuration or
898  *	initialization to support acceleration of FCoE traffic.
899  *
900  * int (*ndo_fcoe_disable)(struct net_device *dev);
901  *	Called when the FCoE protocol stack wants to stop using LLD for FCoE
902  *	so the underlying device can perform whatever needed clean-ups to
903  *	stop supporting acceleration of FCoE traffic.
904  *
905  * int (*ndo_fcoe_ddp_setup)(struct net_device *dev, u16 xid,
906  *			     struct scatterlist *sgl, unsigned int sgc);
907  *	Called when the FCoE Initiator wants to initialize an I/O that
908  *	is a possible candidate for Direct Data Placement (DDP). The LLD can
909  *	perform necessary setup and returns 1 to indicate the device is set up
910  *	successfully to perform DDP on this I/O, otherwise this returns 0.
911  *
912  * int (*ndo_fcoe_ddp_done)(struct net_device *dev,  u16 xid);
913  *	Called when the FCoE Initiator/Target is done with the DDPed I/O as
914  *	indicated by the FC exchange id 'xid', so the underlying device can
915  *	clean up and reuse resources for later DDP requests.
916  *
917  * int (*ndo_fcoe_ddp_target)(struct net_device *dev, u16 xid,
918  *			      struct scatterlist *sgl, unsigned int sgc);
919  *	Called when the FCoE Target wants to initialize an I/O that
920  *	is a possible candidate for Direct Data Placement (DDP). The LLD can
921  *	perform necessary setup and returns 1 to indicate the device is set up
922  *	successfully to perform DDP on this I/O, otherwise this returns 0.
923  *
924  * int (*ndo_fcoe_get_hbainfo)(struct net_device *dev,
925  *			       struct netdev_fcoe_hbainfo *hbainfo);
926  *	Called when the FCoE Protocol stack wants information on the underlying
927  *	device. This information is utilized by the FCoE protocol stack to
928  *	register attributes with Fiber Channel management service as per the
929  *	FC-GS Fabric Device Management Information(FDMI) specification.
930  *
931  * int (*ndo_fcoe_get_wwn)(struct net_device *dev, u64 *wwn, int type);
932  *	Called when the underlying device wants to override default World Wide
933  *	Name (WWN) generation mechanism in FCoE protocol stack to pass its own
934  *	World Wide Port Name (WWPN) or World Wide Node Name (WWNN) to the FCoE
935  *	protocol stack to use.
936  *
937  *	RFS acceleration.
938  * int (*ndo_rx_flow_steer)(struct net_device *dev, const struct sk_buff *skb,
939  *			    u16 rxq_index, u32 flow_id);
940  *	Set hardware filter for RFS.  rxq_index is the target queue index;
941  *	flow_id is a flow ID to be passed to rps_may_expire_flow() later.
942  *	Return the filter ID on success, or a negative error code.
943  *
944  *	Slave management functions (for bridge, bonding, etc).
945  * int (*ndo_add_slave)(struct net_device *dev, struct net_device *slave_dev);
946  *	Called to make another netdev an underling.
947  *
948  * int (*ndo_del_slave)(struct net_device *dev, struct net_device *slave_dev);
949  *	Called to release previously enslaved netdev.
950  *
951  *      Feature/offload setting functions.
952  * netdev_features_t (*ndo_fix_features)(struct net_device *dev,
953  *		netdev_features_t features);
954  *	Adjusts the requested feature flags according to device-specific
955  *	constraints, and returns the resulting flags. Must not modify
956  *	the device state.
957  *
958  * int (*ndo_set_features)(struct net_device *dev, netdev_features_t features);
959  *	Called to update device configuration to new features. Passed
960  *	feature set might be less than what was returned by ndo_fix_features()).
961  *	Must return >0 or -errno if it changed dev->features itself.
962  *
963  * int (*ndo_fdb_add)(struct ndmsg *ndm, struct nlattr *tb[],
964  *		      struct net_device *dev,
965  *		      const unsigned char *addr, u16 vid, u16 flags)
966  *	Adds an FDB entry to dev for addr.
967  * int (*ndo_fdb_del)(struct ndmsg *ndm, struct nlattr *tb[],
968  *		      struct net_device *dev,
969  *		      const unsigned char *addr, u16 vid)
970  *	Deletes the FDB entry from dev coresponding to addr.
971  * int (*ndo_fdb_dump)(struct sk_buff *skb, struct netlink_callback *cb,
972  *		       struct net_device *dev, struct net_device *filter_dev,
973  *		       int idx)
974  *	Used to add FDB entries to dump requests. Implementers should add
975  *	entries to skb and update idx with the number of entries.
976  *
977  * int (*ndo_bridge_setlink)(struct net_device *dev, struct nlmsghdr *nlh,
978  *			     u16 flags)
979  * int (*ndo_bridge_getlink)(struct sk_buff *skb, u32 pid, u32 seq,
980  *			     struct net_device *dev, u32 filter_mask,
981  *			     int nlflags)
982  * int (*ndo_bridge_dellink)(struct net_device *dev, struct nlmsghdr *nlh,
983  *			     u16 flags);
984  *
985  * int (*ndo_change_carrier)(struct net_device *dev, bool new_carrier);
986  *	Called to change device carrier. Soft-devices (like dummy, team, etc)
987  *	which do not represent real hardware may define this to allow their
988  *	userspace components to manage their virtual carrier state. Devices
989  *	that determine carrier state from physical hardware properties (eg
990  *	network cables) or protocol-dependent mechanisms (eg
991  *	USB_CDC_NOTIFY_NETWORK_CONNECTION) should NOT implement this function.
992  *
993  * int (*ndo_get_phys_port_id)(struct net_device *dev,
994  *			       struct netdev_phys_item_id *ppid);
995  *	Called to get ID of physical port of this device. If driver does
996  *	not implement this, it is assumed that the hw is not able to have
997  *	multiple net devices on single physical port.
998  *
999  * void (*ndo_add_vxlan_port)(struct  net_device *dev,
1000  *			      sa_family_t sa_family, __be16 port);
1001  *	Called by vxlan to notiy a driver about the UDP port and socket
1002  *	address family that vxlan is listnening to. It is called only when
1003  *	a new port starts listening. The operation is protected by the
1004  *	vxlan_net->sock_lock.
1005  *
1006  * void (*ndo_del_vxlan_port)(struct  net_device *dev,
1007  *			      sa_family_t sa_family, __be16 port);
1008  *	Called by vxlan to notify the driver about a UDP port and socket
1009  *	address family that vxlan is not listening to anymore. The operation
1010  *	is protected by the vxlan_net->sock_lock.
1011  *
1012  * void* (*ndo_dfwd_add_station)(struct net_device *pdev,
1013  *				 struct net_device *dev)
1014  *	Called by upper layer devices to accelerate switching or other
1015  *	station functionality into hardware. 'pdev is the lowerdev
1016  *	to use for the offload and 'dev' is the net device that will
1017  *	back the offload. Returns a pointer to the private structure
1018  *	the upper layer will maintain.
1019  * void (*ndo_dfwd_del_station)(struct net_device *pdev, void *priv)
1020  *	Called by upper layer device to delete the station created
1021  *	by 'ndo_dfwd_add_station'. 'pdev' is the net device backing
1022  *	the station and priv is the structure returned by the add
1023  *	operation.
1024  * netdev_tx_t (*ndo_dfwd_start_xmit)(struct sk_buff *skb,
1025  *				      struct net_device *dev,
1026  *				      void *priv);
1027  *	Callback to use for xmit over the accelerated station. This
1028  *	is used in place of ndo_start_xmit on accelerated net
1029  *	devices.
1030  * netdev_features_t (*ndo_features_check) (struct sk_buff *skb,
1031  *					    struct net_device *dev
1032  *					    netdev_features_t features);
1033  *	Called by core transmit path to determine if device is capable of
1034  *	performing offload operations on a given packet. This is to give
1035  *	the device an opportunity to implement any restrictions that cannot
1036  *	be otherwise expressed by feature flags. The check is called with
1037  *	the set of features that the stack has calculated and it returns
1038  *	those the driver believes to be appropriate.
1039  * int (*ndo_set_tx_maxrate)(struct net_device *dev,
1040  *			     int queue_index, u32 maxrate);
1041  *	Called when a user wants to set a max-rate limitation of specific
1042  *	TX queue.
1043  * int (*ndo_get_iflink)(const struct net_device *dev);
1044  *	Called to get the iflink value of this device.
1045  */
1046 struct net_device_ops {
1047 	int			(*ndo_init)(struct net_device *dev);
1048 	void			(*ndo_uninit)(struct net_device *dev);
1049 	int			(*ndo_open)(struct net_device *dev);
1050 	int			(*ndo_stop)(struct net_device *dev);
1051 	netdev_tx_t		(*ndo_start_xmit) (struct sk_buff *skb,
1052 						   struct net_device *dev);
1053 	u16			(*ndo_select_queue)(struct net_device *dev,
1054 						    struct sk_buff *skb,
1055 						    void *accel_priv,
1056 						    select_queue_fallback_t fallback);
1057 	void			(*ndo_change_rx_flags)(struct net_device *dev,
1058 						       int flags);
1059 	void			(*ndo_set_rx_mode)(struct net_device *dev);
1060 	int			(*ndo_set_mac_address)(struct net_device *dev,
1061 						       void *addr);
1062 	int			(*ndo_validate_addr)(struct net_device *dev);
1063 	int			(*ndo_do_ioctl)(struct net_device *dev,
1064 					        struct ifreq *ifr, int cmd);
1065 	int			(*ndo_set_config)(struct net_device *dev,
1066 					          struct ifmap *map);
1067 	int			(*ndo_change_mtu)(struct net_device *dev,
1068 						  int new_mtu);
1069 	int			(*ndo_neigh_setup)(struct net_device *dev,
1070 						   struct neigh_parms *);
1071 	void			(*ndo_tx_timeout) (struct net_device *dev);
1072 
1073 	struct rtnl_link_stats64* (*ndo_get_stats64)(struct net_device *dev,
1074 						     struct rtnl_link_stats64 *storage);
1075 	struct net_device_stats* (*ndo_get_stats)(struct net_device *dev);
1076 
1077 	int			(*ndo_vlan_rx_add_vid)(struct net_device *dev,
1078 						       __be16 proto, u16 vid);
1079 	int			(*ndo_vlan_rx_kill_vid)(struct net_device *dev,
1080 						        __be16 proto, u16 vid);
1081 #ifdef CONFIG_NET_POLL_CONTROLLER
1082 	void                    (*ndo_poll_controller)(struct net_device *dev);
1083 	int			(*ndo_netpoll_setup)(struct net_device *dev,
1084 						     struct netpoll_info *info);
1085 	void			(*ndo_netpoll_cleanup)(struct net_device *dev);
1086 #endif
1087 #ifdef CONFIG_NET_RX_BUSY_POLL
1088 	int			(*ndo_busy_poll)(struct napi_struct *dev);
1089 #endif
1090 	int			(*ndo_set_vf_mac)(struct net_device *dev,
1091 						  int queue, u8 *mac);
1092 	int			(*ndo_set_vf_vlan)(struct net_device *dev,
1093 						   int queue, u16 vlan, u8 qos);
1094 	int			(*ndo_set_vf_rate)(struct net_device *dev,
1095 						   int vf, int min_tx_rate,
1096 						   int max_tx_rate);
1097 	int			(*ndo_set_vf_spoofchk)(struct net_device *dev,
1098 						       int vf, bool setting);
1099 	int			(*ndo_get_vf_config)(struct net_device *dev,
1100 						     int vf,
1101 						     struct ifla_vf_info *ivf);
1102 	int			(*ndo_set_vf_link_state)(struct net_device *dev,
1103 							 int vf, int link_state);
1104 	int			(*ndo_set_vf_port)(struct net_device *dev,
1105 						   int vf,
1106 						   struct nlattr *port[]);
1107 	int			(*ndo_get_vf_port)(struct net_device *dev,
1108 						   int vf, struct sk_buff *skb);
1109 	int			(*ndo_set_vf_rss_query_en)(
1110 						   struct net_device *dev,
1111 						   int vf, bool setting);
1112 	int			(*ndo_setup_tc)(struct net_device *dev, u8 tc);
1113 #if IS_ENABLED(CONFIG_FCOE)
1114 	int			(*ndo_fcoe_enable)(struct net_device *dev);
1115 	int			(*ndo_fcoe_disable)(struct net_device *dev);
1116 	int			(*ndo_fcoe_ddp_setup)(struct net_device *dev,
1117 						      u16 xid,
1118 						      struct scatterlist *sgl,
1119 						      unsigned int sgc);
1120 	int			(*ndo_fcoe_ddp_done)(struct net_device *dev,
1121 						     u16 xid);
1122 	int			(*ndo_fcoe_ddp_target)(struct net_device *dev,
1123 						       u16 xid,
1124 						       struct scatterlist *sgl,
1125 						       unsigned int sgc);
1126 	int			(*ndo_fcoe_get_hbainfo)(struct net_device *dev,
1127 							struct netdev_fcoe_hbainfo *hbainfo);
1128 #endif
1129 
1130 #if IS_ENABLED(CONFIG_LIBFCOE)
1131 #define NETDEV_FCOE_WWNN 0
1132 #define NETDEV_FCOE_WWPN 1
1133 	int			(*ndo_fcoe_get_wwn)(struct net_device *dev,
1134 						    u64 *wwn, int type);
1135 #endif
1136 
1137 #ifdef CONFIG_RFS_ACCEL
1138 	int			(*ndo_rx_flow_steer)(struct net_device *dev,
1139 						     const struct sk_buff *skb,
1140 						     u16 rxq_index,
1141 						     u32 flow_id);
1142 #endif
1143 	int			(*ndo_add_slave)(struct net_device *dev,
1144 						 struct net_device *slave_dev);
1145 	int			(*ndo_del_slave)(struct net_device *dev,
1146 						 struct net_device *slave_dev);
1147 	netdev_features_t	(*ndo_fix_features)(struct net_device *dev,
1148 						    netdev_features_t features);
1149 	int			(*ndo_set_features)(struct net_device *dev,
1150 						    netdev_features_t features);
1151 	int			(*ndo_neigh_construct)(struct neighbour *n);
1152 	void			(*ndo_neigh_destroy)(struct neighbour *n);
1153 
1154 	int			(*ndo_fdb_add)(struct ndmsg *ndm,
1155 					       struct nlattr *tb[],
1156 					       struct net_device *dev,
1157 					       const unsigned char *addr,
1158 					       u16 vid,
1159 					       u16 flags);
1160 	int			(*ndo_fdb_del)(struct ndmsg *ndm,
1161 					       struct nlattr *tb[],
1162 					       struct net_device *dev,
1163 					       const unsigned char *addr,
1164 					       u16 vid);
1165 	int			(*ndo_fdb_dump)(struct sk_buff *skb,
1166 						struct netlink_callback *cb,
1167 						struct net_device *dev,
1168 						struct net_device *filter_dev,
1169 						int idx);
1170 
1171 	int			(*ndo_bridge_setlink)(struct net_device *dev,
1172 						      struct nlmsghdr *nlh,
1173 						      u16 flags);
1174 	int			(*ndo_bridge_getlink)(struct sk_buff *skb,
1175 						      u32 pid, u32 seq,
1176 						      struct net_device *dev,
1177 						      u32 filter_mask,
1178 						      int nlflags);
1179 	int			(*ndo_bridge_dellink)(struct net_device *dev,
1180 						      struct nlmsghdr *nlh,
1181 						      u16 flags);
1182 	int			(*ndo_change_carrier)(struct net_device *dev,
1183 						      bool new_carrier);
1184 	int			(*ndo_get_phys_port_id)(struct net_device *dev,
1185 							struct netdev_phys_item_id *ppid);
1186 	int			(*ndo_get_phys_port_name)(struct net_device *dev,
1187 							  char *name, size_t len);
1188 	void			(*ndo_add_vxlan_port)(struct  net_device *dev,
1189 						      sa_family_t sa_family,
1190 						      __be16 port);
1191 	void			(*ndo_del_vxlan_port)(struct  net_device *dev,
1192 						      sa_family_t sa_family,
1193 						      __be16 port);
1194 
1195 	void*			(*ndo_dfwd_add_station)(struct net_device *pdev,
1196 							struct net_device *dev);
1197 	void			(*ndo_dfwd_del_station)(struct net_device *pdev,
1198 							void *priv);
1199 
1200 	netdev_tx_t		(*ndo_dfwd_start_xmit) (struct sk_buff *skb,
1201 							struct net_device *dev,
1202 							void *priv);
1203 	int			(*ndo_get_lock_subclass)(struct net_device *dev);
1204 	netdev_features_t	(*ndo_features_check) (struct sk_buff *skb,
1205 						       struct net_device *dev,
1206 						       netdev_features_t features);
1207 	int			(*ndo_set_tx_maxrate)(struct net_device *dev,
1208 						      int queue_index,
1209 						      u32 maxrate);
1210 	int			(*ndo_get_iflink)(const struct net_device *dev);
1211 };
1212 
1213 /**
1214  * enum net_device_priv_flags - &struct net_device priv_flags
1215  *
1216  * These are the &struct net_device, they are only set internally
1217  * by drivers and used in the kernel. These flags are invisible to
1218  * userspace, this means that the order of these flags can change
1219  * during any kernel release.
1220  *
1221  * You should have a pretty good reason to be extending these flags.
1222  *
1223  * @IFF_802_1Q_VLAN: 802.1Q VLAN device
1224  * @IFF_EBRIDGE: Ethernet bridging device
1225  * @IFF_SLAVE_INACTIVE: bonding slave not the curr. active
1226  * @IFF_MASTER_8023AD: bonding master, 802.3ad
1227  * @IFF_MASTER_ALB: bonding master, balance-alb
1228  * @IFF_BONDING: bonding master or slave
1229  * @IFF_SLAVE_NEEDARP: need ARPs for validation
1230  * @IFF_ISATAP: ISATAP interface (RFC4214)
1231  * @IFF_MASTER_ARPMON: bonding master, ARP mon in use
1232  * @IFF_WAN_HDLC: WAN HDLC device
1233  * @IFF_XMIT_DST_RELEASE: dev_hard_start_xmit() is allowed to
1234  *	release skb->dst
1235  * @IFF_DONT_BRIDGE: disallow bridging this ether dev
1236  * @IFF_DISABLE_NETPOLL: disable netpoll at run-time
1237  * @IFF_MACVLAN_PORT: device used as macvlan port
1238  * @IFF_BRIDGE_PORT: device used as bridge port
1239  * @IFF_OVS_DATAPATH: device used as Open vSwitch datapath port
1240  * @IFF_TX_SKB_SHARING: The interface supports sharing skbs on transmit
1241  * @IFF_UNICAST_FLT: Supports unicast filtering
1242  * @IFF_TEAM_PORT: device used as team port
1243  * @IFF_SUPP_NOFCS: device supports sending custom FCS
1244  * @IFF_LIVE_ADDR_CHANGE: device supports hardware address
1245  *	change when it's running
1246  * @IFF_MACVLAN: Macvlan device
1247  */
1248 enum netdev_priv_flags {
1249 	IFF_802_1Q_VLAN			= 1<<0,
1250 	IFF_EBRIDGE			= 1<<1,
1251 	IFF_SLAVE_INACTIVE		= 1<<2,
1252 	IFF_MASTER_8023AD		= 1<<3,
1253 	IFF_MASTER_ALB			= 1<<4,
1254 	IFF_BONDING			= 1<<5,
1255 	IFF_SLAVE_NEEDARP		= 1<<6,
1256 	IFF_ISATAP			= 1<<7,
1257 	IFF_MASTER_ARPMON		= 1<<8,
1258 	IFF_WAN_HDLC			= 1<<9,
1259 	IFF_XMIT_DST_RELEASE		= 1<<10,
1260 	IFF_DONT_BRIDGE			= 1<<11,
1261 	IFF_DISABLE_NETPOLL		= 1<<12,
1262 	IFF_MACVLAN_PORT		= 1<<13,
1263 	IFF_BRIDGE_PORT			= 1<<14,
1264 	IFF_OVS_DATAPATH		= 1<<15,
1265 	IFF_TX_SKB_SHARING		= 1<<16,
1266 	IFF_UNICAST_FLT			= 1<<17,
1267 	IFF_TEAM_PORT			= 1<<18,
1268 	IFF_SUPP_NOFCS			= 1<<19,
1269 	IFF_LIVE_ADDR_CHANGE		= 1<<20,
1270 	IFF_MACVLAN			= 1<<21,
1271 	IFF_XMIT_DST_RELEASE_PERM	= 1<<22,
1272 	IFF_IPVLAN_MASTER		= 1<<23,
1273 	IFF_IPVLAN_SLAVE		= 1<<24,
1274 };
1275 
1276 #define IFF_802_1Q_VLAN			IFF_802_1Q_VLAN
1277 #define IFF_EBRIDGE			IFF_EBRIDGE
1278 #define IFF_SLAVE_INACTIVE		IFF_SLAVE_INACTIVE
1279 #define IFF_MASTER_8023AD		IFF_MASTER_8023AD
1280 #define IFF_MASTER_ALB			IFF_MASTER_ALB
1281 #define IFF_BONDING			IFF_BONDING
1282 #define IFF_SLAVE_NEEDARP		IFF_SLAVE_NEEDARP
1283 #define IFF_ISATAP			IFF_ISATAP
1284 #define IFF_MASTER_ARPMON		IFF_MASTER_ARPMON
1285 #define IFF_WAN_HDLC			IFF_WAN_HDLC
1286 #define IFF_XMIT_DST_RELEASE		IFF_XMIT_DST_RELEASE
1287 #define IFF_DONT_BRIDGE			IFF_DONT_BRIDGE
1288 #define IFF_DISABLE_NETPOLL		IFF_DISABLE_NETPOLL
1289 #define IFF_MACVLAN_PORT		IFF_MACVLAN_PORT
1290 #define IFF_BRIDGE_PORT			IFF_BRIDGE_PORT
1291 #define IFF_OVS_DATAPATH		IFF_OVS_DATAPATH
1292 #define IFF_TX_SKB_SHARING		IFF_TX_SKB_SHARING
1293 #define IFF_UNICAST_FLT			IFF_UNICAST_FLT
1294 #define IFF_TEAM_PORT			IFF_TEAM_PORT
1295 #define IFF_SUPP_NOFCS			IFF_SUPP_NOFCS
1296 #define IFF_LIVE_ADDR_CHANGE		IFF_LIVE_ADDR_CHANGE
1297 #define IFF_MACVLAN			IFF_MACVLAN
1298 #define IFF_XMIT_DST_RELEASE_PERM	IFF_XMIT_DST_RELEASE_PERM
1299 #define IFF_IPVLAN_MASTER		IFF_IPVLAN_MASTER
1300 #define IFF_IPVLAN_SLAVE		IFF_IPVLAN_SLAVE
1301 
1302 /**
1303  *	struct net_device - The DEVICE structure.
1304  *		Actually, this whole structure is a big mistake.  It mixes I/O
1305  *		data with strictly "high-level" data, and it has to know about
1306  *		almost every data structure used in the INET module.
1307  *
1308  *	@name:	This is the first field of the "visible" part of this structure
1309  *		(i.e. as seen by users in the "Space.c" file).  It is the name
1310  *	 	of the interface.
1311  *
1312  *	@name_hlist: 	Device name hash chain, please keep it close to name[]
1313  *	@ifalias:	SNMP alias
1314  *	@mem_end:	Shared memory end
1315  *	@mem_start:	Shared memory start
1316  *	@base_addr:	Device I/O address
1317  *	@irq:		Device IRQ number
1318  *
1319  *	@carrier_changes:	Stats to monitor carrier on<->off transitions
1320  *
1321  *	@state:		Generic network queuing layer state, see netdev_state_t
1322  *	@dev_list:	The global list of network devices
1323  *	@napi_list:	List entry, that is used for polling napi devices
1324  *	@unreg_list:	List entry, that is used, when we are unregistering the
1325  *			device, see the function unregister_netdev
1326  *	@close_list:	List entry, that is used, when we are closing the device
1327  *
1328  *	@adj_list:	Directly linked devices, like slaves for bonding
1329  *	@all_adj_list:	All linked devices, *including* neighbours
1330  *	@features:	Currently active device features
1331  *	@hw_features:	User-changeable features
1332  *
1333  *	@wanted_features:	User-requested features
1334  *	@vlan_features:		Mask of features inheritable by VLAN devices
1335  *
1336  *	@hw_enc_features:	Mask of features inherited by encapsulating devices
1337  *				This field indicates what encapsulation
1338  *				offloads the hardware is capable of doing,
1339  *				and drivers will need to set them appropriately.
1340  *
1341  *	@mpls_features:	Mask of features inheritable by MPLS
1342  *
1343  *	@ifindex:	interface index
1344  *	@group:		The group, that the device belongs to
1345  *
1346  *	@stats:		Statistics struct, which was left as a legacy, use
1347  *			rtnl_link_stats64 instead
1348  *
1349  *	@rx_dropped:	Dropped packets by core network,
1350  *			do not use this in drivers
1351  *	@tx_dropped:	Dropped packets by core network,
1352  *			do not use this in drivers
1353  *
1354  *	@wireless_handlers:	List of functions to handle Wireless Extensions,
1355  *				instead of ioctl,
1356  *				see <net/iw_handler.h> for details.
1357  *	@wireless_data:	Instance data managed by the core of wireless extensions
1358  *
1359  *	@netdev_ops:	Includes several pointers to callbacks,
1360  *			if one wants to override the ndo_*() functions
1361  *	@ethtool_ops:	Management operations
1362  *	@header_ops:	Includes callbacks for creating,parsing,caching,etc
1363  *			of Layer 2 headers.
1364  *
1365  *	@flags:		Interface flags (a la BSD)
1366  *	@priv_flags:	Like 'flags' but invisible to userspace,
1367  *			see if.h for the definitions
1368  *	@gflags:	Global flags ( kept as legacy )
1369  *	@padded:	How much padding added by alloc_netdev()
1370  *	@operstate:	RFC2863 operstate
1371  *	@link_mode:	Mapping policy to operstate
1372  *	@if_port:	Selectable AUI, TP, ...
1373  *	@dma:		DMA channel
1374  *	@mtu:		Interface MTU value
1375  *	@type:		Interface hardware type
1376  *	@hard_header_len: Hardware header length
1377  *
1378  *	@needed_headroom: Extra headroom the hardware may need, but not in all
1379  *			  cases can this be guaranteed
1380  *	@needed_tailroom: Extra tailroom the hardware may need, but not in all
1381  *			  cases can this be guaranteed. Some cases also use
1382  *			  LL_MAX_HEADER instead to allocate the skb
1383  *
1384  *	interface address info:
1385  *
1386  * 	@perm_addr:		Permanent hw address
1387  * 	@addr_assign_type:	Hw address assignment type
1388  * 	@addr_len:		Hardware address length
1389  * 	@neigh_priv_len;	Used in neigh_alloc(),
1390  * 				initialized only in atm/clip.c
1391  * 	@dev_id:		Used to differentiate devices that share
1392  * 				the same link layer address
1393  * 	@dev_port:		Used to differentiate devices that share
1394  * 				the same function
1395  *	@addr_list_lock:	XXX: need comments on this one
1396  *	@uc_promisc:		Counter, that indicates, that promiscuous mode
1397  *				has been enabled due to the need to listen to
1398  *				additional unicast addresses in a device that
1399  *				does not implement ndo_set_rx_mode()
1400  *	@uc:			unicast mac addresses
1401  *	@mc:			multicast mac addresses
1402  *	@dev_addrs:		list of device hw addresses
1403  *	@queues_kset:		Group of all Kobjects in the Tx and RX queues
1404  *	@promiscuity:		Number of times, the NIC is told to work in
1405  *				Promiscuous mode, if it becomes 0 the NIC will
1406  *				exit from working in Promiscuous mode
1407  *	@allmulti:		Counter, enables or disables allmulticast mode
1408  *
1409  *	@vlan_info:	VLAN info
1410  *	@dsa_ptr:	dsa specific data
1411  *	@tipc_ptr:	TIPC specific data
1412  *	@atalk_ptr:	AppleTalk link
1413  *	@ip_ptr:	IPv4 specific data
1414  *	@dn_ptr:	DECnet specific data
1415  *	@ip6_ptr:	IPv6 specific data
1416  *	@ax25_ptr:	AX.25 specific data
1417  *	@ieee80211_ptr:	IEEE 802.11 specific data, assign before registering
1418  *
1419  *	@last_rx:	Time of last Rx
1420  *	@dev_addr:	Hw address (before bcast,
1421  *			because most packets are unicast)
1422  *
1423  *	@_rx:			Array of RX queues
1424  *	@num_rx_queues:		Number of RX queues
1425  *				allocated at register_netdev() time
1426  *	@real_num_rx_queues: 	Number of RX queues currently active in device
1427  *
1428  *	@rx_handler:		handler for received packets
1429  *	@rx_handler_data: 	XXX: need comments on this one
1430  *	@ingress_queue:		XXX: need comments on this one
1431  *	@broadcast:		hw bcast address
1432  *
1433  *	@rx_cpu_rmap:	CPU reverse-mapping for RX completion interrupts,
1434  *			indexed by RX queue number. Assigned by driver.
1435  *			This must only be set if the ndo_rx_flow_steer
1436  *			operation is defined
1437  *	@index_hlist:		Device index hash chain
1438  *
1439  *	@_tx:			Array of TX queues
1440  *	@num_tx_queues:		Number of TX queues allocated at alloc_netdev_mq() time
1441  *	@real_num_tx_queues: 	Number of TX queues currently active in device
1442  *	@qdisc:			Root qdisc from userspace point of view
1443  *	@tx_queue_len:		Max frames per queue allowed
1444  *	@tx_global_lock: 	XXX: need comments on this one
1445  *
1446  *	@xps_maps:	XXX: need comments on this one
1447  *
1448  *	@trans_start:		Time (in jiffies) of last Tx
1449  *	@watchdog_timeo:	Represents the timeout that is used by
1450  *				the watchdog ( see dev_watchdog() )
1451  *	@watchdog_timer:	List of timers
1452  *
1453  *	@pcpu_refcnt:		Number of references to this device
1454  *	@todo_list:		Delayed register/unregister
1455  *	@link_watch_list:	XXX: need comments on this one
1456  *
1457  *	@reg_state:		Register/unregister state machine
1458  *	@dismantle:		Device is going to be freed
1459  *	@rtnl_link_state:	This enum represents the phases of creating
1460  *				a new link
1461  *
1462  *	@destructor:		Called from unregister,
1463  *				can be used to call free_netdev
1464  *	@npinfo:		XXX: need comments on this one
1465  * 	@nd_net:		Network namespace this network device is inside
1466  *
1467  * 	@ml_priv:	Mid-layer private
1468  * 	@lstats:	Loopback statistics
1469  * 	@tstats:	Tunnel statistics
1470  * 	@dstats:	Dummy statistics
1471  * 	@vstats:	Virtual ethernet statistics
1472  *
1473  *	@garp_port:	GARP
1474  *	@mrp_port:	MRP
1475  *
1476  *	@dev:		Class/net/name entry
1477  *	@sysfs_groups:	Space for optional device, statistics and wireless
1478  *			sysfs groups
1479  *
1480  *	@sysfs_rx_queue_group:	Space for optional per-rx queue attributes
1481  *	@rtnl_link_ops:	Rtnl_link_ops
1482  *
1483  *	@gso_max_size:	Maximum size of generic segmentation offload
1484  *	@gso_max_segs:	Maximum number of segments that can be passed to the
1485  *			NIC for GSO
1486  *	@gso_min_segs:	Minimum number of segments that can be passed to the
1487  *			NIC for GSO
1488  *
1489  *	@dcbnl_ops:	Data Center Bridging netlink ops
1490  *	@num_tc:	Number of traffic classes in the net device
1491  *	@tc_to_txq:	XXX: need comments on this one
1492  *	@prio_tc_map	XXX: need comments on this one
1493  *
1494  *	@fcoe_ddp_xid:	Max exchange id for FCoE LRO by ddp
1495  *
1496  *	@priomap:	XXX: need comments on this one
1497  *	@phydev:	Physical device may attach itself
1498  *			for hardware timestamping
1499  *
1500  *	@qdisc_tx_busylock:	XXX: need comments on this one
1501  *
1502  *	@pm_qos_req:	Power Management QoS object
1503  *
1504  *	FIXME: cleanup struct net_device such that network protocol info
1505  *	moves out.
1506  */
1507 
1508 struct net_device {
1509 	char			name[IFNAMSIZ];
1510 	struct hlist_node	name_hlist;
1511 	char 			*ifalias;
1512 	/*
1513 	 *	I/O specific fields
1514 	 *	FIXME: Merge these and struct ifmap into one
1515 	 */
1516 	unsigned long		mem_end;
1517 	unsigned long		mem_start;
1518 	unsigned long		base_addr;
1519 	int			irq;
1520 
1521 	atomic_t		carrier_changes;
1522 
1523 	/*
1524 	 *	Some hardware also needs these fields (state,dev_list,
1525 	 *	napi_list,unreg_list,close_list) but they are not
1526 	 *	part of the usual set specified in Space.c.
1527 	 */
1528 
1529 	unsigned long		state;
1530 
1531 	struct list_head	dev_list;
1532 	struct list_head	napi_list;
1533 	struct list_head	unreg_list;
1534 	struct list_head	close_list;
1535 	struct list_head	ptype_all;
1536 	struct list_head	ptype_specific;
1537 
1538 	struct {
1539 		struct list_head upper;
1540 		struct list_head lower;
1541 	} adj_list;
1542 
1543 	struct {
1544 		struct list_head upper;
1545 		struct list_head lower;
1546 	} all_adj_list;
1547 
1548 	netdev_features_t	features;
1549 	netdev_features_t	hw_features;
1550 	netdev_features_t	wanted_features;
1551 	netdev_features_t	vlan_features;
1552 	netdev_features_t	hw_enc_features;
1553 	netdev_features_t	mpls_features;
1554 
1555 	int			ifindex;
1556 	int			group;
1557 
1558 	struct net_device_stats	stats;
1559 
1560 	atomic_long_t		rx_dropped;
1561 	atomic_long_t		tx_dropped;
1562 
1563 #ifdef CONFIG_WIRELESS_EXT
1564 	const struct iw_handler_def *	wireless_handlers;
1565 	struct iw_public_data *	wireless_data;
1566 #endif
1567 	const struct net_device_ops *netdev_ops;
1568 	const struct ethtool_ops *ethtool_ops;
1569 #ifdef CONFIG_NET_SWITCHDEV
1570 	const struct swdev_ops *swdev_ops;
1571 #endif
1572 
1573 	const struct header_ops *header_ops;
1574 
1575 	unsigned int		flags;
1576 	unsigned int		priv_flags;
1577 
1578 	unsigned short		gflags;
1579 	unsigned short		padded;
1580 
1581 	unsigned char		operstate;
1582 	unsigned char		link_mode;
1583 
1584 	unsigned char		if_port;
1585 	unsigned char		dma;
1586 
1587 	unsigned int		mtu;
1588 	unsigned short		type;
1589 	unsigned short		hard_header_len;
1590 
1591 	unsigned short		needed_headroom;
1592 	unsigned short		needed_tailroom;
1593 
1594 	/* Interface address info. */
1595 	unsigned char		perm_addr[MAX_ADDR_LEN];
1596 	unsigned char		addr_assign_type;
1597 	unsigned char		addr_len;
1598 	unsigned short		neigh_priv_len;
1599 	unsigned short          dev_id;
1600 	unsigned short          dev_port;
1601 	spinlock_t		addr_list_lock;
1602 	unsigned char		name_assign_type;
1603 	bool			uc_promisc;
1604 	struct netdev_hw_addr_list	uc;
1605 	struct netdev_hw_addr_list	mc;
1606 	struct netdev_hw_addr_list	dev_addrs;
1607 
1608 #ifdef CONFIG_SYSFS
1609 	struct kset		*queues_kset;
1610 #endif
1611 	unsigned int		promiscuity;
1612 	unsigned int		allmulti;
1613 
1614 
1615 	/* Protocol specific pointers */
1616 
1617 #if IS_ENABLED(CONFIG_VLAN_8021Q)
1618 	struct vlan_info __rcu	*vlan_info;
1619 #endif
1620 #if IS_ENABLED(CONFIG_NET_DSA)
1621 	struct dsa_switch_tree	*dsa_ptr;
1622 #endif
1623 #if IS_ENABLED(CONFIG_TIPC)
1624 	struct tipc_bearer __rcu *tipc_ptr;
1625 #endif
1626 	void 			*atalk_ptr;
1627 	struct in_device __rcu	*ip_ptr;
1628 	struct dn_dev __rcu     *dn_ptr;
1629 	struct inet6_dev __rcu	*ip6_ptr;
1630 	void			*ax25_ptr;
1631 	struct wireless_dev	*ieee80211_ptr;
1632 	struct wpan_dev		*ieee802154_ptr;
1633 #if IS_ENABLED(CONFIG_MPLS_ROUTING)
1634 	struct mpls_dev __rcu	*mpls_ptr;
1635 #endif
1636 
1637 /*
1638  * Cache lines mostly used on receive path (including eth_type_trans())
1639  */
1640 	unsigned long		last_rx;
1641 
1642 	/* Interface address info used in eth_type_trans() */
1643 	unsigned char		*dev_addr;
1644 
1645 
1646 #ifdef CONFIG_SYSFS
1647 	struct netdev_rx_queue	*_rx;
1648 
1649 	unsigned int		num_rx_queues;
1650 	unsigned int		real_num_rx_queues;
1651 
1652 #endif
1653 
1654 	unsigned long		gro_flush_timeout;
1655 	rx_handler_func_t __rcu	*rx_handler;
1656 	void __rcu		*rx_handler_data;
1657 
1658 	struct netdev_queue __rcu *ingress_queue;
1659 	unsigned char		broadcast[MAX_ADDR_LEN];
1660 #ifdef CONFIG_RFS_ACCEL
1661 	struct cpu_rmap		*rx_cpu_rmap;
1662 #endif
1663 	struct hlist_node	index_hlist;
1664 
1665 /*
1666  * Cache lines mostly used on transmit path
1667  */
1668 	struct netdev_queue	*_tx ____cacheline_aligned_in_smp;
1669 	unsigned int		num_tx_queues;
1670 	unsigned int		real_num_tx_queues;
1671 	struct Qdisc		*qdisc;
1672 	unsigned long		tx_queue_len;
1673 	spinlock_t		tx_global_lock;
1674 	int			watchdog_timeo;
1675 
1676 #ifdef CONFIG_XPS
1677 	struct xps_dev_maps __rcu *xps_maps;
1678 #endif
1679 
1680 	/* These may be needed for future network-power-down code. */
1681 
1682 	/*
1683 	 * trans_start here is expensive for high speed devices on SMP,
1684 	 * please use netdev_queue->trans_start instead.
1685 	 */
1686 	unsigned long		trans_start;
1687 
1688 	struct timer_list	watchdog_timer;
1689 
1690 	int __percpu		*pcpu_refcnt;
1691 	struct list_head	todo_list;
1692 
1693 	struct list_head	link_watch_list;
1694 
1695 	enum { NETREG_UNINITIALIZED=0,
1696 	       NETREG_REGISTERED,	/* completed register_netdevice */
1697 	       NETREG_UNREGISTERING,	/* called unregister_netdevice */
1698 	       NETREG_UNREGISTERED,	/* completed unregister todo */
1699 	       NETREG_RELEASED,		/* called free_netdev */
1700 	       NETREG_DUMMY,		/* dummy device for NAPI poll */
1701 	} reg_state:8;
1702 
1703 	bool dismantle;
1704 
1705 	enum {
1706 		RTNL_LINK_INITIALIZED,
1707 		RTNL_LINK_INITIALIZING,
1708 	} rtnl_link_state:16;
1709 
1710 	void (*destructor)(struct net_device *dev);
1711 
1712 #ifdef CONFIG_NETPOLL
1713 	struct netpoll_info __rcu	*npinfo;
1714 #endif
1715 
1716 	possible_net_t			nd_net;
1717 
1718 	/* mid-layer private */
1719 	union {
1720 		void					*ml_priv;
1721 		struct pcpu_lstats __percpu		*lstats;
1722 		struct pcpu_sw_netstats __percpu	*tstats;
1723 		struct pcpu_dstats __percpu		*dstats;
1724 		struct pcpu_vstats __percpu		*vstats;
1725 	};
1726 
1727 	struct garp_port __rcu	*garp_port;
1728 	struct mrp_port __rcu	*mrp_port;
1729 
1730 	struct device	dev;
1731 	const struct attribute_group *sysfs_groups[4];
1732 	const struct attribute_group *sysfs_rx_queue_group;
1733 
1734 	const struct rtnl_link_ops *rtnl_link_ops;
1735 
1736 	/* for setting kernel sock attribute on TCP connection setup */
1737 #define GSO_MAX_SIZE		65536
1738 	unsigned int		gso_max_size;
1739 #define GSO_MAX_SEGS		65535
1740 	u16			gso_max_segs;
1741 	u16			gso_min_segs;
1742 #ifdef CONFIG_DCB
1743 	const struct dcbnl_rtnl_ops *dcbnl_ops;
1744 #endif
1745 	u8 num_tc;
1746 	struct netdev_tc_txq tc_to_txq[TC_MAX_QUEUE];
1747 	u8 prio_tc_map[TC_BITMASK + 1];
1748 
1749 #if IS_ENABLED(CONFIG_FCOE)
1750 	unsigned int		fcoe_ddp_xid;
1751 #endif
1752 #if IS_ENABLED(CONFIG_CGROUP_NET_PRIO)
1753 	struct netprio_map __rcu *priomap;
1754 #endif
1755 	struct phy_device *phydev;
1756 	struct lock_class_key *qdisc_tx_busylock;
1757 };
1758 #define to_net_dev(d) container_of(d, struct net_device, dev)
1759 
1760 #define	NETDEV_ALIGN		32
1761 
1762 static inline
1763 int netdev_get_prio_tc_map(const struct net_device *dev, u32 prio)
1764 {
1765 	return dev->prio_tc_map[prio & TC_BITMASK];
1766 }
1767 
1768 static inline
1769 int netdev_set_prio_tc_map(struct net_device *dev, u8 prio, u8 tc)
1770 {
1771 	if (tc >= dev->num_tc)
1772 		return -EINVAL;
1773 
1774 	dev->prio_tc_map[prio & TC_BITMASK] = tc & TC_BITMASK;
1775 	return 0;
1776 }
1777 
1778 static inline
1779 void netdev_reset_tc(struct net_device *dev)
1780 {
1781 	dev->num_tc = 0;
1782 	memset(dev->tc_to_txq, 0, sizeof(dev->tc_to_txq));
1783 	memset(dev->prio_tc_map, 0, sizeof(dev->prio_tc_map));
1784 }
1785 
1786 static inline
1787 int netdev_set_tc_queue(struct net_device *dev, u8 tc, u16 count, u16 offset)
1788 {
1789 	if (tc >= dev->num_tc)
1790 		return -EINVAL;
1791 
1792 	dev->tc_to_txq[tc].count = count;
1793 	dev->tc_to_txq[tc].offset = offset;
1794 	return 0;
1795 }
1796 
1797 static inline
1798 int netdev_set_num_tc(struct net_device *dev, u8 num_tc)
1799 {
1800 	if (num_tc > TC_MAX_QUEUE)
1801 		return -EINVAL;
1802 
1803 	dev->num_tc = num_tc;
1804 	return 0;
1805 }
1806 
1807 static inline
1808 int netdev_get_num_tc(struct net_device *dev)
1809 {
1810 	return dev->num_tc;
1811 }
1812 
1813 static inline
1814 struct netdev_queue *netdev_get_tx_queue(const struct net_device *dev,
1815 					 unsigned int index)
1816 {
1817 	return &dev->_tx[index];
1818 }
1819 
1820 static inline struct netdev_queue *skb_get_tx_queue(const struct net_device *dev,
1821 						    const struct sk_buff *skb)
1822 {
1823 	return netdev_get_tx_queue(dev, skb_get_queue_mapping(skb));
1824 }
1825 
1826 static inline void netdev_for_each_tx_queue(struct net_device *dev,
1827 					    void (*f)(struct net_device *,
1828 						      struct netdev_queue *,
1829 						      void *),
1830 					    void *arg)
1831 {
1832 	unsigned int i;
1833 
1834 	for (i = 0; i < dev->num_tx_queues; i++)
1835 		f(dev, &dev->_tx[i], arg);
1836 }
1837 
1838 struct netdev_queue *netdev_pick_tx(struct net_device *dev,
1839 				    struct sk_buff *skb,
1840 				    void *accel_priv);
1841 
1842 /*
1843  * Net namespace inlines
1844  */
1845 static inline
1846 struct net *dev_net(const struct net_device *dev)
1847 {
1848 	return read_pnet(&dev->nd_net);
1849 }
1850 
1851 static inline
1852 void dev_net_set(struct net_device *dev, struct net *net)
1853 {
1854 	write_pnet(&dev->nd_net, net);
1855 }
1856 
1857 static inline bool netdev_uses_dsa(struct net_device *dev)
1858 {
1859 #if IS_ENABLED(CONFIG_NET_DSA)
1860 	if (dev->dsa_ptr != NULL)
1861 		return dsa_uses_tagged_protocol(dev->dsa_ptr);
1862 #endif
1863 	return false;
1864 }
1865 
1866 /**
1867  *	netdev_priv - access network device private data
1868  *	@dev: network device
1869  *
1870  * Get network device private data
1871  */
1872 static inline void *netdev_priv(const struct net_device *dev)
1873 {
1874 	return (char *)dev + ALIGN(sizeof(struct net_device), NETDEV_ALIGN);
1875 }
1876 
1877 /* Set the sysfs physical device reference for the network logical device
1878  * if set prior to registration will cause a symlink during initialization.
1879  */
1880 #define SET_NETDEV_DEV(net, pdev)	((net)->dev.parent = (pdev))
1881 
1882 /* Set the sysfs device type for the network logical device to allow
1883  * fine-grained identification of different network device types. For
1884  * example Ethernet, Wirelss LAN, Bluetooth, WiMAX etc.
1885  */
1886 #define SET_NETDEV_DEVTYPE(net, devtype)	((net)->dev.type = (devtype))
1887 
1888 /* Default NAPI poll() weight
1889  * Device drivers are strongly advised to not use bigger value
1890  */
1891 #define NAPI_POLL_WEIGHT 64
1892 
1893 /**
1894  *	netif_napi_add - initialize a napi context
1895  *	@dev:  network device
1896  *	@napi: napi context
1897  *	@poll: polling function
1898  *	@weight: default weight
1899  *
1900  * netif_napi_add() must be used to initialize a napi context prior to calling
1901  * *any* of the other napi related functions.
1902  */
1903 void netif_napi_add(struct net_device *dev, struct napi_struct *napi,
1904 		    int (*poll)(struct napi_struct *, int), int weight);
1905 
1906 /**
1907  *  netif_napi_del - remove a napi context
1908  *  @napi: napi context
1909  *
1910  *  netif_napi_del() removes a napi context from the network device napi list
1911  */
1912 void netif_napi_del(struct napi_struct *napi);
1913 
1914 struct napi_gro_cb {
1915 	/* Virtual address of skb_shinfo(skb)->frags[0].page + offset. */
1916 	void *frag0;
1917 
1918 	/* Length of frag0. */
1919 	unsigned int frag0_len;
1920 
1921 	/* This indicates where we are processing relative to skb->data. */
1922 	int data_offset;
1923 
1924 	/* This is non-zero if the packet cannot be merged with the new skb. */
1925 	u16	flush;
1926 
1927 	/* Save the IP ID here and check when we get to the transport layer */
1928 	u16	flush_id;
1929 
1930 	/* Number of segments aggregated. */
1931 	u16	count;
1932 
1933 	/* Start offset for remote checksum offload */
1934 	u16	gro_remcsum_start;
1935 
1936 	/* jiffies when first packet was created/queued */
1937 	unsigned long age;
1938 
1939 	/* Used in ipv6_gro_receive() and foo-over-udp */
1940 	u16	proto;
1941 
1942 	/* This is non-zero if the packet may be of the same flow. */
1943 	u8	same_flow:1;
1944 
1945 	/* Used in udp_gro_receive */
1946 	u8	udp_mark:1;
1947 
1948 	/* GRO checksum is valid */
1949 	u8	csum_valid:1;
1950 
1951 	/* Number of checksums via CHECKSUM_UNNECESSARY */
1952 	u8	csum_cnt:3;
1953 
1954 	/* Free the skb? */
1955 	u8	free:2;
1956 #define NAPI_GRO_FREE		  1
1957 #define NAPI_GRO_FREE_STOLEN_HEAD 2
1958 
1959 	/* Used in foo-over-udp, set in udp[46]_gro_receive */
1960 	u8	is_ipv6:1;
1961 
1962 	/* 7 bit hole */
1963 
1964 	/* used to support CHECKSUM_COMPLETE for tunneling protocols */
1965 	__wsum	csum;
1966 
1967 	/* used in skb_gro_receive() slow path */
1968 	struct sk_buff *last;
1969 };
1970 
1971 #define NAPI_GRO_CB(skb) ((struct napi_gro_cb *)(skb)->cb)
1972 
1973 struct packet_type {
1974 	__be16			type;	/* This is really htons(ether_type). */
1975 	struct net_device	*dev;	/* NULL is wildcarded here	     */
1976 	int			(*func) (struct sk_buff *,
1977 					 struct net_device *,
1978 					 struct packet_type *,
1979 					 struct net_device *);
1980 	bool			(*id_match)(struct packet_type *ptype,
1981 					    struct sock *sk);
1982 	void			*af_packet_priv;
1983 	struct list_head	list;
1984 };
1985 
1986 struct offload_callbacks {
1987 	struct sk_buff		*(*gso_segment)(struct sk_buff *skb,
1988 						netdev_features_t features);
1989 	struct sk_buff		**(*gro_receive)(struct sk_buff **head,
1990 						 struct sk_buff *skb);
1991 	int			(*gro_complete)(struct sk_buff *skb, int nhoff);
1992 };
1993 
1994 struct packet_offload {
1995 	__be16			 type;	/* This is really htons(ether_type). */
1996 	struct offload_callbacks callbacks;
1997 	struct list_head	 list;
1998 };
1999 
2000 struct udp_offload;
2001 
2002 struct udp_offload_callbacks {
2003 	struct sk_buff		**(*gro_receive)(struct sk_buff **head,
2004 						 struct sk_buff *skb,
2005 						 struct udp_offload *uoff);
2006 	int			(*gro_complete)(struct sk_buff *skb,
2007 						int nhoff,
2008 						struct udp_offload *uoff);
2009 };
2010 
2011 struct udp_offload {
2012 	__be16			 port;
2013 	u8			 ipproto;
2014 	struct udp_offload_callbacks callbacks;
2015 };
2016 
2017 /* often modified stats are per cpu, other are shared (netdev->stats) */
2018 struct pcpu_sw_netstats {
2019 	u64     rx_packets;
2020 	u64     rx_bytes;
2021 	u64     tx_packets;
2022 	u64     tx_bytes;
2023 	struct u64_stats_sync   syncp;
2024 };
2025 
2026 #define netdev_alloc_pcpu_stats(type)				\
2027 ({								\
2028 	typeof(type) __percpu *pcpu_stats = alloc_percpu(type); \
2029 	if (pcpu_stats)	{					\
2030 		int __cpu;					\
2031 		for_each_possible_cpu(__cpu) {			\
2032 			typeof(type) *stat;			\
2033 			stat = per_cpu_ptr(pcpu_stats, __cpu);	\
2034 			u64_stats_init(&stat->syncp);		\
2035 		}						\
2036 	}							\
2037 	pcpu_stats;						\
2038 })
2039 
2040 #include <linux/notifier.h>
2041 
2042 /* netdevice notifier chain. Please remember to update the rtnetlink
2043  * notification exclusion list in rtnetlink_event() when adding new
2044  * types.
2045  */
2046 #define NETDEV_UP	0x0001	/* For now you can't veto a device up/down */
2047 #define NETDEV_DOWN	0x0002
2048 #define NETDEV_REBOOT	0x0003	/* Tell a protocol stack a network interface
2049 				   detected a hardware crash and restarted
2050 				   - we can use this eg to kick tcp sessions
2051 				   once done */
2052 #define NETDEV_CHANGE	0x0004	/* Notify device state change */
2053 #define NETDEV_REGISTER 0x0005
2054 #define NETDEV_UNREGISTER	0x0006
2055 #define NETDEV_CHANGEMTU	0x0007 /* notify after mtu change happened */
2056 #define NETDEV_CHANGEADDR	0x0008
2057 #define NETDEV_GOING_DOWN	0x0009
2058 #define NETDEV_CHANGENAME	0x000A
2059 #define NETDEV_FEAT_CHANGE	0x000B
2060 #define NETDEV_BONDING_FAILOVER 0x000C
2061 #define NETDEV_PRE_UP		0x000D
2062 #define NETDEV_PRE_TYPE_CHANGE	0x000E
2063 #define NETDEV_POST_TYPE_CHANGE	0x000F
2064 #define NETDEV_POST_INIT	0x0010
2065 #define NETDEV_UNREGISTER_FINAL 0x0011
2066 #define NETDEV_RELEASE		0x0012
2067 #define NETDEV_NOTIFY_PEERS	0x0013
2068 #define NETDEV_JOIN		0x0014
2069 #define NETDEV_CHANGEUPPER	0x0015
2070 #define NETDEV_RESEND_IGMP	0x0016
2071 #define NETDEV_PRECHANGEMTU	0x0017 /* notify before mtu change happened */
2072 #define NETDEV_CHANGEINFODATA	0x0018
2073 #define NETDEV_BONDING_INFO	0x0019
2074 
2075 int register_netdevice_notifier(struct notifier_block *nb);
2076 int unregister_netdevice_notifier(struct notifier_block *nb);
2077 
2078 struct netdev_notifier_info {
2079 	struct net_device *dev;
2080 };
2081 
2082 struct netdev_notifier_change_info {
2083 	struct netdev_notifier_info info; /* must be first */
2084 	unsigned int flags_changed;
2085 };
2086 
2087 static inline void netdev_notifier_info_init(struct netdev_notifier_info *info,
2088 					     struct net_device *dev)
2089 {
2090 	info->dev = dev;
2091 }
2092 
2093 static inline struct net_device *
2094 netdev_notifier_info_to_dev(const struct netdev_notifier_info *info)
2095 {
2096 	return info->dev;
2097 }
2098 
2099 int call_netdevice_notifiers(unsigned long val, struct net_device *dev);
2100 
2101 
2102 extern rwlock_t				dev_base_lock;		/* Device list lock */
2103 
2104 #define for_each_netdev(net, d)		\
2105 		list_for_each_entry(d, &(net)->dev_base_head, dev_list)
2106 #define for_each_netdev_reverse(net, d)	\
2107 		list_for_each_entry_reverse(d, &(net)->dev_base_head, dev_list)
2108 #define for_each_netdev_rcu(net, d)		\
2109 		list_for_each_entry_rcu(d, &(net)->dev_base_head, dev_list)
2110 #define for_each_netdev_safe(net, d, n)	\
2111 		list_for_each_entry_safe(d, n, &(net)->dev_base_head, dev_list)
2112 #define for_each_netdev_continue(net, d)		\
2113 		list_for_each_entry_continue(d, &(net)->dev_base_head, dev_list)
2114 #define for_each_netdev_continue_rcu(net, d)		\
2115 	list_for_each_entry_continue_rcu(d, &(net)->dev_base_head, dev_list)
2116 #define for_each_netdev_in_bond_rcu(bond, slave)	\
2117 		for_each_netdev_rcu(&init_net, slave)	\
2118 			if (netdev_master_upper_dev_get_rcu(slave) == (bond))
2119 #define net_device_entry(lh)	list_entry(lh, struct net_device, dev_list)
2120 
2121 static inline struct net_device *next_net_device(struct net_device *dev)
2122 {
2123 	struct list_head *lh;
2124 	struct net *net;
2125 
2126 	net = dev_net(dev);
2127 	lh = dev->dev_list.next;
2128 	return lh == &net->dev_base_head ? NULL : net_device_entry(lh);
2129 }
2130 
2131 static inline struct net_device *next_net_device_rcu(struct net_device *dev)
2132 {
2133 	struct list_head *lh;
2134 	struct net *net;
2135 
2136 	net = dev_net(dev);
2137 	lh = rcu_dereference(list_next_rcu(&dev->dev_list));
2138 	return lh == &net->dev_base_head ? NULL : net_device_entry(lh);
2139 }
2140 
2141 static inline struct net_device *first_net_device(struct net *net)
2142 {
2143 	return list_empty(&net->dev_base_head) ? NULL :
2144 		net_device_entry(net->dev_base_head.next);
2145 }
2146 
2147 static inline struct net_device *first_net_device_rcu(struct net *net)
2148 {
2149 	struct list_head *lh = rcu_dereference(list_next_rcu(&net->dev_base_head));
2150 
2151 	return lh == &net->dev_base_head ? NULL : net_device_entry(lh);
2152 }
2153 
2154 int netdev_boot_setup_check(struct net_device *dev);
2155 unsigned long netdev_boot_base(const char *prefix, int unit);
2156 struct net_device *dev_getbyhwaddr_rcu(struct net *net, unsigned short type,
2157 				       const char *hwaddr);
2158 struct net_device *dev_getfirstbyhwtype(struct net *net, unsigned short type);
2159 struct net_device *__dev_getfirstbyhwtype(struct net *net, unsigned short type);
2160 void dev_add_pack(struct packet_type *pt);
2161 void dev_remove_pack(struct packet_type *pt);
2162 void __dev_remove_pack(struct packet_type *pt);
2163 void dev_add_offload(struct packet_offload *po);
2164 void dev_remove_offload(struct packet_offload *po);
2165 
2166 int dev_get_iflink(const struct net_device *dev);
2167 struct net_device *__dev_get_by_flags(struct net *net, unsigned short flags,
2168 				      unsigned short mask);
2169 struct net_device *dev_get_by_name(struct net *net, const char *name);
2170 struct net_device *dev_get_by_name_rcu(struct net *net, const char *name);
2171 struct net_device *__dev_get_by_name(struct net *net, const char *name);
2172 int dev_alloc_name(struct net_device *dev, const char *name);
2173 int dev_open(struct net_device *dev);
2174 int dev_close(struct net_device *dev);
2175 int dev_close_many(struct list_head *head, bool unlink);
2176 void dev_disable_lro(struct net_device *dev);
2177 int dev_loopback_xmit(struct sock *sk, struct sk_buff *newskb);
2178 int dev_queue_xmit_sk(struct sock *sk, struct sk_buff *skb);
2179 static inline int dev_queue_xmit(struct sk_buff *skb)
2180 {
2181 	return dev_queue_xmit_sk(skb->sk, skb);
2182 }
2183 int dev_queue_xmit_accel(struct sk_buff *skb, void *accel_priv);
2184 int register_netdevice(struct net_device *dev);
2185 void unregister_netdevice_queue(struct net_device *dev, struct list_head *head);
2186 void unregister_netdevice_many(struct list_head *head);
2187 static inline void unregister_netdevice(struct net_device *dev)
2188 {
2189 	unregister_netdevice_queue(dev, NULL);
2190 }
2191 
2192 int netdev_refcnt_read(const struct net_device *dev);
2193 void free_netdev(struct net_device *dev);
2194 void netdev_freemem(struct net_device *dev);
2195 void synchronize_net(void);
2196 int init_dummy_netdev(struct net_device *dev);
2197 
2198 DECLARE_PER_CPU(int, xmit_recursion);
2199 static inline int dev_recursion_level(void)
2200 {
2201 	return this_cpu_read(xmit_recursion);
2202 }
2203 
2204 struct net_device *dev_get_by_index(struct net *net, int ifindex);
2205 struct net_device *__dev_get_by_index(struct net *net, int ifindex);
2206 struct net_device *dev_get_by_index_rcu(struct net *net, int ifindex);
2207 int netdev_get_name(struct net *net, char *name, int ifindex);
2208 int dev_restart(struct net_device *dev);
2209 int skb_gro_receive(struct sk_buff **head, struct sk_buff *skb);
2210 
2211 static inline unsigned int skb_gro_offset(const struct sk_buff *skb)
2212 {
2213 	return NAPI_GRO_CB(skb)->data_offset;
2214 }
2215 
2216 static inline unsigned int skb_gro_len(const struct sk_buff *skb)
2217 {
2218 	return skb->len - NAPI_GRO_CB(skb)->data_offset;
2219 }
2220 
2221 static inline void skb_gro_pull(struct sk_buff *skb, unsigned int len)
2222 {
2223 	NAPI_GRO_CB(skb)->data_offset += len;
2224 }
2225 
2226 static inline void *skb_gro_header_fast(struct sk_buff *skb,
2227 					unsigned int offset)
2228 {
2229 	return NAPI_GRO_CB(skb)->frag0 + offset;
2230 }
2231 
2232 static inline int skb_gro_header_hard(struct sk_buff *skb, unsigned int hlen)
2233 {
2234 	return NAPI_GRO_CB(skb)->frag0_len < hlen;
2235 }
2236 
2237 static inline void *skb_gro_header_slow(struct sk_buff *skb, unsigned int hlen,
2238 					unsigned int offset)
2239 {
2240 	if (!pskb_may_pull(skb, hlen))
2241 		return NULL;
2242 
2243 	NAPI_GRO_CB(skb)->frag0 = NULL;
2244 	NAPI_GRO_CB(skb)->frag0_len = 0;
2245 	return skb->data + offset;
2246 }
2247 
2248 static inline void *skb_gro_network_header(struct sk_buff *skb)
2249 {
2250 	return (NAPI_GRO_CB(skb)->frag0 ?: skb->data) +
2251 	       skb_network_offset(skb);
2252 }
2253 
2254 static inline void skb_gro_postpull_rcsum(struct sk_buff *skb,
2255 					const void *start, unsigned int len)
2256 {
2257 	if (NAPI_GRO_CB(skb)->csum_valid)
2258 		NAPI_GRO_CB(skb)->csum = csum_sub(NAPI_GRO_CB(skb)->csum,
2259 						  csum_partial(start, len, 0));
2260 }
2261 
2262 /* GRO checksum functions. These are logical equivalents of the normal
2263  * checksum functions (in skbuff.h) except that they operate on the GRO
2264  * offsets and fields in sk_buff.
2265  */
2266 
2267 __sum16 __skb_gro_checksum_complete(struct sk_buff *skb);
2268 
2269 static inline bool skb_at_gro_remcsum_start(struct sk_buff *skb)
2270 {
2271 	return (NAPI_GRO_CB(skb)->gro_remcsum_start - skb_headroom(skb) ==
2272 		skb_gro_offset(skb));
2273 }
2274 
2275 static inline bool __skb_gro_checksum_validate_needed(struct sk_buff *skb,
2276 						      bool zero_okay,
2277 						      __sum16 check)
2278 {
2279 	return ((skb->ip_summed != CHECKSUM_PARTIAL ||
2280 		skb_checksum_start_offset(skb) <
2281 		 skb_gro_offset(skb)) &&
2282 		!skb_at_gro_remcsum_start(skb) &&
2283 		NAPI_GRO_CB(skb)->csum_cnt == 0 &&
2284 		(!zero_okay || check));
2285 }
2286 
2287 static inline __sum16 __skb_gro_checksum_validate_complete(struct sk_buff *skb,
2288 							   __wsum psum)
2289 {
2290 	if (NAPI_GRO_CB(skb)->csum_valid &&
2291 	    !csum_fold(csum_add(psum, NAPI_GRO_CB(skb)->csum)))
2292 		return 0;
2293 
2294 	NAPI_GRO_CB(skb)->csum = psum;
2295 
2296 	return __skb_gro_checksum_complete(skb);
2297 }
2298 
2299 static inline void skb_gro_incr_csum_unnecessary(struct sk_buff *skb)
2300 {
2301 	if (NAPI_GRO_CB(skb)->csum_cnt > 0) {
2302 		/* Consume a checksum from CHECKSUM_UNNECESSARY */
2303 		NAPI_GRO_CB(skb)->csum_cnt--;
2304 	} else {
2305 		/* Update skb for CHECKSUM_UNNECESSARY and csum_level when we
2306 		 * verified a new top level checksum or an encapsulated one
2307 		 * during GRO. This saves work if we fallback to normal path.
2308 		 */
2309 		__skb_incr_checksum_unnecessary(skb);
2310 	}
2311 }
2312 
2313 #define __skb_gro_checksum_validate(skb, proto, zero_okay, check,	\
2314 				    compute_pseudo)			\
2315 ({									\
2316 	__sum16 __ret = 0;						\
2317 	if (__skb_gro_checksum_validate_needed(skb, zero_okay, check))	\
2318 		__ret = __skb_gro_checksum_validate_complete(skb,	\
2319 				compute_pseudo(skb, proto));		\
2320 	if (__ret)							\
2321 		__skb_mark_checksum_bad(skb);				\
2322 	else								\
2323 		skb_gro_incr_csum_unnecessary(skb);			\
2324 	__ret;								\
2325 })
2326 
2327 #define skb_gro_checksum_validate(skb, proto, compute_pseudo)		\
2328 	__skb_gro_checksum_validate(skb, proto, false, 0, compute_pseudo)
2329 
2330 #define skb_gro_checksum_validate_zero_check(skb, proto, check,		\
2331 					     compute_pseudo)		\
2332 	__skb_gro_checksum_validate(skb, proto, true, check, compute_pseudo)
2333 
2334 #define skb_gro_checksum_simple_validate(skb)				\
2335 	__skb_gro_checksum_validate(skb, 0, false, 0, null_compute_pseudo)
2336 
2337 static inline bool __skb_gro_checksum_convert_check(struct sk_buff *skb)
2338 {
2339 	return (NAPI_GRO_CB(skb)->csum_cnt == 0 &&
2340 		!NAPI_GRO_CB(skb)->csum_valid);
2341 }
2342 
2343 static inline void __skb_gro_checksum_convert(struct sk_buff *skb,
2344 					      __sum16 check, __wsum pseudo)
2345 {
2346 	NAPI_GRO_CB(skb)->csum = ~pseudo;
2347 	NAPI_GRO_CB(skb)->csum_valid = 1;
2348 }
2349 
2350 #define skb_gro_checksum_try_convert(skb, proto, check, compute_pseudo)	\
2351 do {									\
2352 	if (__skb_gro_checksum_convert_check(skb))			\
2353 		__skb_gro_checksum_convert(skb, check,			\
2354 					   compute_pseudo(skb, proto));	\
2355 } while (0)
2356 
2357 struct gro_remcsum {
2358 	int offset;
2359 	__wsum delta;
2360 };
2361 
2362 static inline void skb_gro_remcsum_init(struct gro_remcsum *grc)
2363 {
2364 	grc->offset = 0;
2365 	grc->delta = 0;
2366 }
2367 
2368 static inline void skb_gro_remcsum_process(struct sk_buff *skb, void *ptr,
2369 					   int start, int offset,
2370 					   struct gro_remcsum *grc,
2371 					   bool nopartial)
2372 {
2373 	__wsum delta;
2374 
2375 	BUG_ON(!NAPI_GRO_CB(skb)->csum_valid);
2376 
2377 	if (!nopartial) {
2378 		NAPI_GRO_CB(skb)->gro_remcsum_start =
2379 		    ((unsigned char *)ptr + start) - skb->head;
2380 		return;
2381 	}
2382 
2383 	delta = remcsum_adjust(ptr, NAPI_GRO_CB(skb)->csum, start, offset);
2384 
2385 	/* Adjust skb->csum since we changed the packet */
2386 	NAPI_GRO_CB(skb)->csum = csum_add(NAPI_GRO_CB(skb)->csum, delta);
2387 
2388 	grc->offset = (ptr + offset) - (void *)skb->head;
2389 	grc->delta = delta;
2390 }
2391 
2392 static inline void skb_gro_remcsum_cleanup(struct sk_buff *skb,
2393 					   struct gro_remcsum *grc)
2394 {
2395 	if (!grc->delta)
2396 		return;
2397 
2398 	remcsum_unadjust((__sum16 *)(skb->head + grc->offset), grc->delta);
2399 }
2400 
2401 static inline int dev_hard_header(struct sk_buff *skb, struct net_device *dev,
2402 				  unsigned short type,
2403 				  const void *daddr, const void *saddr,
2404 				  unsigned int len)
2405 {
2406 	if (!dev->header_ops || !dev->header_ops->create)
2407 		return 0;
2408 
2409 	return dev->header_ops->create(skb, dev, type, daddr, saddr, len);
2410 }
2411 
2412 static inline int dev_parse_header(const struct sk_buff *skb,
2413 				   unsigned char *haddr)
2414 {
2415 	const struct net_device *dev = skb->dev;
2416 
2417 	if (!dev->header_ops || !dev->header_ops->parse)
2418 		return 0;
2419 	return dev->header_ops->parse(skb, haddr);
2420 }
2421 
2422 typedef int gifconf_func_t(struct net_device * dev, char __user * bufptr, int len);
2423 int register_gifconf(unsigned int family, gifconf_func_t *gifconf);
2424 static inline int unregister_gifconf(unsigned int family)
2425 {
2426 	return register_gifconf(family, NULL);
2427 }
2428 
2429 #ifdef CONFIG_NET_FLOW_LIMIT
2430 #define FLOW_LIMIT_HISTORY	(1 << 7)  /* must be ^2 and !overflow buckets */
2431 struct sd_flow_limit {
2432 	u64			count;
2433 	unsigned int		num_buckets;
2434 	unsigned int		history_head;
2435 	u16			history[FLOW_LIMIT_HISTORY];
2436 	u8			buckets[];
2437 };
2438 
2439 extern int netdev_flow_limit_table_len;
2440 #endif /* CONFIG_NET_FLOW_LIMIT */
2441 
2442 /*
2443  * Incoming packets are placed on per-cpu queues
2444  */
2445 struct softnet_data {
2446 	struct list_head	poll_list;
2447 	struct sk_buff_head	process_queue;
2448 
2449 	/* stats */
2450 	unsigned int		processed;
2451 	unsigned int		time_squeeze;
2452 	unsigned int		cpu_collision;
2453 	unsigned int		received_rps;
2454 #ifdef CONFIG_RPS
2455 	struct softnet_data	*rps_ipi_list;
2456 #endif
2457 #ifdef CONFIG_NET_FLOW_LIMIT
2458 	struct sd_flow_limit __rcu *flow_limit;
2459 #endif
2460 	struct Qdisc		*output_queue;
2461 	struct Qdisc		**output_queue_tailp;
2462 	struct sk_buff		*completion_queue;
2463 
2464 #ifdef CONFIG_RPS
2465 	/* Elements below can be accessed between CPUs for RPS */
2466 	struct call_single_data	csd ____cacheline_aligned_in_smp;
2467 	struct softnet_data	*rps_ipi_next;
2468 	unsigned int		cpu;
2469 	unsigned int		input_queue_head;
2470 	unsigned int		input_queue_tail;
2471 #endif
2472 	unsigned int		dropped;
2473 	struct sk_buff_head	input_pkt_queue;
2474 	struct napi_struct	backlog;
2475 
2476 };
2477 
2478 static inline void input_queue_head_incr(struct softnet_data *sd)
2479 {
2480 #ifdef CONFIG_RPS
2481 	sd->input_queue_head++;
2482 #endif
2483 }
2484 
2485 static inline void input_queue_tail_incr_save(struct softnet_data *sd,
2486 					      unsigned int *qtail)
2487 {
2488 #ifdef CONFIG_RPS
2489 	*qtail = ++sd->input_queue_tail;
2490 #endif
2491 }
2492 
2493 DECLARE_PER_CPU_ALIGNED(struct softnet_data, softnet_data);
2494 
2495 void __netif_schedule(struct Qdisc *q);
2496 void netif_schedule_queue(struct netdev_queue *txq);
2497 
2498 static inline void netif_tx_schedule_all(struct net_device *dev)
2499 {
2500 	unsigned int i;
2501 
2502 	for (i = 0; i < dev->num_tx_queues; i++)
2503 		netif_schedule_queue(netdev_get_tx_queue(dev, i));
2504 }
2505 
2506 static inline void netif_tx_start_queue(struct netdev_queue *dev_queue)
2507 {
2508 	clear_bit(__QUEUE_STATE_DRV_XOFF, &dev_queue->state);
2509 }
2510 
2511 /**
2512  *	netif_start_queue - allow transmit
2513  *	@dev: network device
2514  *
2515  *	Allow upper layers to call the device hard_start_xmit routine.
2516  */
2517 static inline void netif_start_queue(struct net_device *dev)
2518 {
2519 	netif_tx_start_queue(netdev_get_tx_queue(dev, 0));
2520 }
2521 
2522 static inline void netif_tx_start_all_queues(struct net_device *dev)
2523 {
2524 	unsigned int i;
2525 
2526 	for (i = 0; i < dev->num_tx_queues; i++) {
2527 		struct netdev_queue *txq = netdev_get_tx_queue(dev, i);
2528 		netif_tx_start_queue(txq);
2529 	}
2530 }
2531 
2532 void netif_tx_wake_queue(struct netdev_queue *dev_queue);
2533 
2534 /**
2535  *	netif_wake_queue - restart transmit
2536  *	@dev: network device
2537  *
2538  *	Allow upper layers to call the device hard_start_xmit routine.
2539  *	Used for flow control when transmit resources are available.
2540  */
2541 static inline void netif_wake_queue(struct net_device *dev)
2542 {
2543 	netif_tx_wake_queue(netdev_get_tx_queue(dev, 0));
2544 }
2545 
2546 static inline void netif_tx_wake_all_queues(struct net_device *dev)
2547 {
2548 	unsigned int i;
2549 
2550 	for (i = 0; i < dev->num_tx_queues; i++) {
2551 		struct netdev_queue *txq = netdev_get_tx_queue(dev, i);
2552 		netif_tx_wake_queue(txq);
2553 	}
2554 }
2555 
2556 static inline void netif_tx_stop_queue(struct netdev_queue *dev_queue)
2557 {
2558 	if (WARN_ON(!dev_queue)) {
2559 		pr_info("netif_stop_queue() cannot be called before register_netdev()\n");
2560 		return;
2561 	}
2562 	set_bit(__QUEUE_STATE_DRV_XOFF, &dev_queue->state);
2563 }
2564 
2565 /**
2566  *	netif_stop_queue - stop transmitted packets
2567  *	@dev: network device
2568  *
2569  *	Stop upper layers calling the device hard_start_xmit routine.
2570  *	Used for flow control when transmit resources are unavailable.
2571  */
2572 static inline void netif_stop_queue(struct net_device *dev)
2573 {
2574 	netif_tx_stop_queue(netdev_get_tx_queue(dev, 0));
2575 }
2576 
2577 static inline void netif_tx_stop_all_queues(struct net_device *dev)
2578 {
2579 	unsigned int i;
2580 
2581 	for (i = 0; i < dev->num_tx_queues; i++) {
2582 		struct netdev_queue *txq = netdev_get_tx_queue(dev, i);
2583 		netif_tx_stop_queue(txq);
2584 	}
2585 }
2586 
2587 static inline bool netif_tx_queue_stopped(const struct netdev_queue *dev_queue)
2588 {
2589 	return test_bit(__QUEUE_STATE_DRV_XOFF, &dev_queue->state);
2590 }
2591 
2592 /**
2593  *	netif_queue_stopped - test if transmit queue is flowblocked
2594  *	@dev: network device
2595  *
2596  *	Test if transmit queue on device is currently unable to send.
2597  */
2598 static inline bool netif_queue_stopped(const struct net_device *dev)
2599 {
2600 	return netif_tx_queue_stopped(netdev_get_tx_queue(dev, 0));
2601 }
2602 
2603 static inline bool netif_xmit_stopped(const struct netdev_queue *dev_queue)
2604 {
2605 	return dev_queue->state & QUEUE_STATE_ANY_XOFF;
2606 }
2607 
2608 static inline bool
2609 netif_xmit_frozen_or_stopped(const struct netdev_queue *dev_queue)
2610 {
2611 	return dev_queue->state & QUEUE_STATE_ANY_XOFF_OR_FROZEN;
2612 }
2613 
2614 static inline bool
2615 netif_xmit_frozen_or_drv_stopped(const struct netdev_queue *dev_queue)
2616 {
2617 	return dev_queue->state & QUEUE_STATE_DRV_XOFF_OR_FROZEN;
2618 }
2619 
2620 /**
2621  *	netdev_txq_bql_enqueue_prefetchw - prefetch bql data for write
2622  *	@dev_queue: pointer to transmit queue
2623  *
2624  * BQL enabled drivers might use this helper in their ndo_start_xmit(),
2625  * to give appropriate hint to the cpu.
2626  */
2627 static inline void netdev_txq_bql_enqueue_prefetchw(struct netdev_queue *dev_queue)
2628 {
2629 #ifdef CONFIG_BQL
2630 	prefetchw(&dev_queue->dql.num_queued);
2631 #endif
2632 }
2633 
2634 /**
2635  *	netdev_txq_bql_complete_prefetchw - prefetch bql data for write
2636  *	@dev_queue: pointer to transmit queue
2637  *
2638  * BQL enabled drivers might use this helper in their TX completion path,
2639  * to give appropriate hint to the cpu.
2640  */
2641 static inline void netdev_txq_bql_complete_prefetchw(struct netdev_queue *dev_queue)
2642 {
2643 #ifdef CONFIG_BQL
2644 	prefetchw(&dev_queue->dql.limit);
2645 #endif
2646 }
2647 
2648 static inline void netdev_tx_sent_queue(struct netdev_queue *dev_queue,
2649 					unsigned int bytes)
2650 {
2651 #ifdef CONFIG_BQL
2652 	dql_queued(&dev_queue->dql, bytes);
2653 
2654 	if (likely(dql_avail(&dev_queue->dql) >= 0))
2655 		return;
2656 
2657 	set_bit(__QUEUE_STATE_STACK_XOFF, &dev_queue->state);
2658 
2659 	/*
2660 	 * The XOFF flag must be set before checking the dql_avail below,
2661 	 * because in netdev_tx_completed_queue we update the dql_completed
2662 	 * before checking the XOFF flag.
2663 	 */
2664 	smp_mb();
2665 
2666 	/* check again in case another CPU has just made room avail */
2667 	if (unlikely(dql_avail(&dev_queue->dql) >= 0))
2668 		clear_bit(__QUEUE_STATE_STACK_XOFF, &dev_queue->state);
2669 #endif
2670 }
2671 
2672 /**
2673  * 	netdev_sent_queue - report the number of bytes queued to hardware
2674  * 	@dev: network device
2675  * 	@bytes: number of bytes queued to the hardware device queue
2676  *
2677  * 	Report the number of bytes queued for sending/completion to the network
2678  * 	device hardware queue. @bytes should be a good approximation and should
2679  * 	exactly match netdev_completed_queue() @bytes
2680  */
2681 static inline void netdev_sent_queue(struct net_device *dev, unsigned int bytes)
2682 {
2683 	netdev_tx_sent_queue(netdev_get_tx_queue(dev, 0), bytes);
2684 }
2685 
2686 static inline void netdev_tx_completed_queue(struct netdev_queue *dev_queue,
2687 					     unsigned int pkts, unsigned int bytes)
2688 {
2689 #ifdef CONFIG_BQL
2690 	if (unlikely(!bytes))
2691 		return;
2692 
2693 	dql_completed(&dev_queue->dql, bytes);
2694 
2695 	/*
2696 	 * Without the memory barrier there is a small possiblity that
2697 	 * netdev_tx_sent_queue will miss the update and cause the queue to
2698 	 * be stopped forever
2699 	 */
2700 	smp_mb();
2701 
2702 	if (dql_avail(&dev_queue->dql) < 0)
2703 		return;
2704 
2705 	if (test_and_clear_bit(__QUEUE_STATE_STACK_XOFF, &dev_queue->state))
2706 		netif_schedule_queue(dev_queue);
2707 #endif
2708 }
2709 
2710 /**
2711  * 	netdev_completed_queue - report bytes and packets completed by device
2712  * 	@dev: network device
2713  * 	@pkts: actual number of packets sent over the medium
2714  * 	@bytes: actual number of bytes sent over the medium
2715  *
2716  * 	Report the number of bytes and packets transmitted by the network device
2717  * 	hardware queue over the physical medium, @bytes must exactly match the
2718  * 	@bytes amount passed to netdev_sent_queue()
2719  */
2720 static inline void netdev_completed_queue(struct net_device *dev,
2721 					  unsigned int pkts, unsigned int bytes)
2722 {
2723 	netdev_tx_completed_queue(netdev_get_tx_queue(dev, 0), pkts, bytes);
2724 }
2725 
2726 static inline void netdev_tx_reset_queue(struct netdev_queue *q)
2727 {
2728 #ifdef CONFIG_BQL
2729 	clear_bit(__QUEUE_STATE_STACK_XOFF, &q->state);
2730 	dql_reset(&q->dql);
2731 #endif
2732 }
2733 
2734 /**
2735  * 	netdev_reset_queue - reset the packets and bytes count of a network device
2736  * 	@dev_queue: network device
2737  *
2738  * 	Reset the bytes and packet count of a network device and clear the
2739  * 	software flow control OFF bit for this network device
2740  */
2741 static inline void netdev_reset_queue(struct net_device *dev_queue)
2742 {
2743 	netdev_tx_reset_queue(netdev_get_tx_queue(dev_queue, 0));
2744 }
2745 
2746 /**
2747  * 	netdev_cap_txqueue - check if selected tx queue exceeds device queues
2748  * 	@dev: network device
2749  * 	@queue_index: given tx queue index
2750  *
2751  * 	Returns 0 if given tx queue index >= number of device tx queues,
2752  * 	otherwise returns the originally passed tx queue index.
2753  */
2754 static inline u16 netdev_cap_txqueue(struct net_device *dev, u16 queue_index)
2755 {
2756 	if (unlikely(queue_index >= dev->real_num_tx_queues)) {
2757 		net_warn_ratelimited("%s selects TX queue %d, but real number of TX queues is %d\n",
2758 				     dev->name, queue_index,
2759 				     dev->real_num_tx_queues);
2760 		return 0;
2761 	}
2762 
2763 	return queue_index;
2764 }
2765 
2766 /**
2767  *	netif_running - test if up
2768  *	@dev: network device
2769  *
2770  *	Test if the device has been brought up.
2771  */
2772 static inline bool netif_running(const struct net_device *dev)
2773 {
2774 	return test_bit(__LINK_STATE_START, &dev->state);
2775 }
2776 
2777 /*
2778  * Routines to manage the subqueues on a device.  We only need start
2779  * stop, and a check if it's stopped.  All other device management is
2780  * done at the overall netdevice level.
2781  * Also test the device if we're multiqueue.
2782  */
2783 
2784 /**
2785  *	netif_start_subqueue - allow sending packets on subqueue
2786  *	@dev: network device
2787  *	@queue_index: sub queue index
2788  *
2789  * Start individual transmit queue of a device with multiple transmit queues.
2790  */
2791 static inline void netif_start_subqueue(struct net_device *dev, u16 queue_index)
2792 {
2793 	struct netdev_queue *txq = netdev_get_tx_queue(dev, queue_index);
2794 
2795 	netif_tx_start_queue(txq);
2796 }
2797 
2798 /**
2799  *	netif_stop_subqueue - stop sending packets on subqueue
2800  *	@dev: network device
2801  *	@queue_index: sub queue index
2802  *
2803  * Stop individual transmit queue of a device with multiple transmit queues.
2804  */
2805 static inline void netif_stop_subqueue(struct net_device *dev, u16 queue_index)
2806 {
2807 	struct netdev_queue *txq = netdev_get_tx_queue(dev, queue_index);
2808 	netif_tx_stop_queue(txq);
2809 }
2810 
2811 /**
2812  *	netif_subqueue_stopped - test status of subqueue
2813  *	@dev: network device
2814  *	@queue_index: sub queue index
2815  *
2816  * Check individual transmit queue of a device with multiple transmit queues.
2817  */
2818 static inline bool __netif_subqueue_stopped(const struct net_device *dev,
2819 					    u16 queue_index)
2820 {
2821 	struct netdev_queue *txq = netdev_get_tx_queue(dev, queue_index);
2822 
2823 	return netif_tx_queue_stopped(txq);
2824 }
2825 
2826 static inline bool netif_subqueue_stopped(const struct net_device *dev,
2827 					  struct sk_buff *skb)
2828 {
2829 	return __netif_subqueue_stopped(dev, skb_get_queue_mapping(skb));
2830 }
2831 
2832 void netif_wake_subqueue(struct net_device *dev, u16 queue_index);
2833 
2834 #ifdef CONFIG_XPS
2835 int netif_set_xps_queue(struct net_device *dev, const struct cpumask *mask,
2836 			u16 index);
2837 #else
2838 static inline int netif_set_xps_queue(struct net_device *dev,
2839 				      const struct cpumask *mask,
2840 				      u16 index)
2841 {
2842 	return 0;
2843 }
2844 #endif
2845 
2846 /*
2847  * Returns a Tx hash for the given packet when dev->real_num_tx_queues is used
2848  * as a distribution range limit for the returned value.
2849  */
2850 static inline u16 skb_tx_hash(const struct net_device *dev,
2851 			      struct sk_buff *skb)
2852 {
2853 	return __skb_tx_hash(dev, skb, dev->real_num_tx_queues);
2854 }
2855 
2856 /**
2857  *	netif_is_multiqueue - test if device has multiple transmit queues
2858  *	@dev: network device
2859  *
2860  * Check if device has multiple transmit queues
2861  */
2862 static inline bool netif_is_multiqueue(const struct net_device *dev)
2863 {
2864 	return dev->num_tx_queues > 1;
2865 }
2866 
2867 int netif_set_real_num_tx_queues(struct net_device *dev, unsigned int txq);
2868 
2869 #ifdef CONFIG_SYSFS
2870 int netif_set_real_num_rx_queues(struct net_device *dev, unsigned int rxq);
2871 #else
2872 static inline int netif_set_real_num_rx_queues(struct net_device *dev,
2873 						unsigned int rxq)
2874 {
2875 	return 0;
2876 }
2877 #endif
2878 
2879 #ifdef CONFIG_SYSFS
2880 static inline unsigned int get_netdev_rx_queue_index(
2881 		struct netdev_rx_queue *queue)
2882 {
2883 	struct net_device *dev = queue->dev;
2884 	int index = queue - dev->_rx;
2885 
2886 	BUG_ON(index >= dev->num_rx_queues);
2887 	return index;
2888 }
2889 #endif
2890 
2891 #define DEFAULT_MAX_NUM_RSS_QUEUES	(8)
2892 int netif_get_num_default_rss_queues(void);
2893 
2894 enum skb_free_reason {
2895 	SKB_REASON_CONSUMED,
2896 	SKB_REASON_DROPPED,
2897 };
2898 
2899 void __dev_kfree_skb_irq(struct sk_buff *skb, enum skb_free_reason reason);
2900 void __dev_kfree_skb_any(struct sk_buff *skb, enum skb_free_reason reason);
2901 
2902 /*
2903  * It is not allowed to call kfree_skb() or consume_skb() from hardware
2904  * interrupt context or with hardware interrupts being disabled.
2905  * (in_irq() || irqs_disabled())
2906  *
2907  * We provide four helpers that can be used in following contexts :
2908  *
2909  * dev_kfree_skb_irq(skb) when caller drops a packet from irq context,
2910  *  replacing kfree_skb(skb)
2911  *
2912  * dev_consume_skb_irq(skb) when caller consumes a packet from irq context.
2913  *  Typically used in place of consume_skb(skb) in TX completion path
2914  *
2915  * dev_kfree_skb_any(skb) when caller doesn't know its current irq context,
2916  *  replacing kfree_skb(skb)
2917  *
2918  * dev_consume_skb_any(skb) when caller doesn't know its current irq context,
2919  *  and consumed a packet. Used in place of consume_skb(skb)
2920  */
2921 static inline void dev_kfree_skb_irq(struct sk_buff *skb)
2922 {
2923 	__dev_kfree_skb_irq(skb, SKB_REASON_DROPPED);
2924 }
2925 
2926 static inline void dev_consume_skb_irq(struct sk_buff *skb)
2927 {
2928 	__dev_kfree_skb_irq(skb, SKB_REASON_CONSUMED);
2929 }
2930 
2931 static inline void dev_kfree_skb_any(struct sk_buff *skb)
2932 {
2933 	__dev_kfree_skb_any(skb, SKB_REASON_DROPPED);
2934 }
2935 
2936 static inline void dev_consume_skb_any(struct sk_buff *skb)
2937 {
2938 	__dev_kfree_skb_any(skb, SKB_REASON_CONSUMED);
2939 }
2940 
2941 int netif_rx(struct sk_buff *skb);
2942 int netif_rx_ni(struct sk_buff *skb);
2943 int netif_receive_skb_sk(struct sock *sk, struct sk_buff *skb);
2944 static inline int netif_receive_skb(struct sk_buff *skb)
2945 {
2946 	return netif_receive_skb_sk(skb->sk, skb);
2947 }
2948 gro_result_t napi_gro_receive(struct napi_struct *napi, struct sk_buff *skb);
2949 void napi_gro_flush(struct napi_struct *napi, bool flush_old);
2950 struct sk_buff *napi_get_frags(struct napi_struct *napi);
2951 gro_result_t napi_gro_frags(struct napi_struct *napi);
2952 struct packet_offload *gro_find_receive_by_type(__be16 type);
2953 struct packet_offload *gro_find_complete_by_type(__be16 type);
2954 
2955 static inline void napi_free_frags(struct napi_struct *napi)
2956 {
2957 	kfree_skb(napi->skb);
2958 	napi->skb = NULL;
2959 }
2960 
2961 int netdev_rx_handler_register(struct net_device *dev,
2962 			       rx_handler_func_t *rx_handler,
2963 			       void *rx_handler_data);
2964 void netdev_rx_handler_unregister(struct net_device *dev);
2965 
2966 bool dev_valid_name(const char *name);
2967 int dev_ioctl(struct net *net, unsigned int cmd, void __user *);
2968 int dev_ethtool(struct net *net, struct ifreq *);
2969 unsigned int dev_get_flags(const struct net_device *);
2970 int __dev_change_flags(struct net_device *, unsigned int flags);
2971 int dev_change_flags(struct net_device *, unsigned int);
2972 void __dev_notify_flags(struct net_device *, unsigned int old_flags,
2973 			unsigned int gchanges);
2974 int dev_change_name(struct net_device *, const char *);
2975 int dev_set_alias(struct net_device *, const char *, size_t);
2976 int dev_change_net_namespace(struct net_device *, struct net *, const char *);
2977 int dev_set_mtu(struct net_device *, int);
2978 void dev_set_group(struct net_device *, int);
2979 int dev_set_mac_address(struct net_device *, struct sockaddr *);
2980 int dev_change_carrier(struct net_device *, bool new_carrier);
2981 int dev_get_phys_port_id(struct net_device *dev,
2982 			 struct netdev_phys_item_id *ppid);
2983 int dev_get_phys_port_name(struct net_device *dev,
2984 			   char *name, size_t len);
2985 struct sk_buff *validate_xmit_skb_list(struct sk_buff *skb, struct net_device *dev);
2986 struct sk_buff *dev_hard_start_xmit(struct sk_buff *skb, struct net_device *dev,
2987 				    struct netdev_queue *txq, int *ret);
2988 int __dev_forward_skb(struct net_device *dev, struct sk_buff *skb);
2989 int dev_forward_skb(struct net_device *dev, struct sk_buff *skb);
2990 bool is_skb_forwardable(struct net_device *dev, struct sk_buff *skb);
2991 
2992 extern int		netdev_budget;
2993 
2994 /* Called by rtnetlink.c:rtnl_unlock() */
2995 void netdev_run_todo(void);
2996 
2997 /**
2998  *	dev_put - release reference to device
2999  *	@dev: network device
3000  *
3001  * Release reference to device to allow it to be freed.
3002  */
3003 static inline void dev_put(struct net_device *dev)
3004 {
3005 	this_cpu_dec(*dev->pcpu_refcnt);
3006 }
3007 
3008 /**
3009  *	dev_hold - get reference to device
3010  *	@dev: network device
3011  *
3012  * Hold reference to device to keep it from being freed.
3013  */
3014 static inline void dev_hold(struct net_device *dev)
3015 {
3016 	this_cpu_inc(*dev->pcpu_refcnt);
3017 }
3018 
3019 /* Carrier loss detection, dial on demand. The functions netif_carrier_on
3020  * and _off may be called from IRQ context, but it is caller
3021  * who is responsible for serialization of these calls.
3022  *
3023  * The name carrier is inappropriate, these functions should really be
3024  * called netif_lowerlayer_*() because they represent the state of any
3025  * kind of lower layer not just hardware media.
3026  */
3027 
3028 void linkwatch_init_dev(struct net_device *dev);
3029 void linkwatch_fire_event(struct net_device *dev);
3030 void linkwatch_forget_dev(struct net_device *dev);
3031 
3032 /**
3033  *	netif_carrier_ok - test if carrier present
3034  *	@dev: network device
3035  *
3036  * Check if carrier is present on device
3037  */
3038 static inline bool netif_carrier_ok(const struct net_device *dev)
3039 {
3040 	return !test_bit(__LINK_STATE_NOCARRIER, &dev->state);
3041 }
3042 
3043 unsigned long dev_trans_start(struct net_device *dev);
3044 
3045 void __netdev_watchdog_up(struct net_device *dev);
3046 
3047 void netif_carrier_on(struct net_device *dev);
3048 
3049 void netif_carrier_off(struct net_device *dev);
3050 
3051 /**
3052  *	netif_dormant_on - mark device as dormant.
3053  *	@dev: network device
3054  *
3055  * Mark device as dormant (as per RFC2863).
3056  *
3057  * The dormant state indicates that the relevant interface is not
3058  * actually in a condition to pass packets (i.e., it is not 'up') but is
3059  * in a "pending" state, waiting for some external event.  For "on-
3060  * demand" interfaces, this new state identifies the situation where the
3061  * interface is waiting for events to place it in the up state.
3062  *
3063  */
3064 static inline void netif_dormant_on(struct net_device *dev)
3065 {
3066 	if (!test_and_set_bit(__LINK_STATE_DORMANT, &dev->state))
3067 		linkwatch_fire_event(dev);
3068 }
3069 
3070 /**
3071  *	netif_dormant_off - set device as not dormant.
3072  *	@dev: network device
3073  *
3074  * Device is not in dormant state.
3075  */
3076 static inline void netif_dormant_off(struct net_device *dev)
3077 {
3078 	if (test_and_clear_bit(__LINK_STATE_DORMANT, &dev->state))
3079 		linkwatch_fire_event(dev);
3080 }
3081 
3082 /**
3083  *	netif_dormant - test if carrier present
3084  *	@dev: network device
3085  *
3086  * Check if carrier is present on device
3087  */
3088 static inline bool netif_dormant(const struct net_device *dev)
3089 {
3090 	return test_bit(__LINK_STATE_DORMANT, &dev->state);
3091 }
3092 
3093 
3094 /**
3095  *	netif_oper_up - test if device is operational
3096  *	@dev: network device
3097  *
3098  * Check if carrier is operational
3099  */
3100 static inline bool netif_oper_up(const struct net_device *dev)
3101 {
3102 	return (dev->operstate == IF_OPER_UP ||
3103 		dev->operstate == IF_OPER_UNKNOWN /* backward compat */);
3104 }
3105 
3106 /**
3107  *	netif_device_present - is device available or removed
3108  *	@dev: network device
3109  *
3110  * Check if device has not been removed from system.
3111  */
3112 static inline bool netif_device_present(struct net_device *dev)
3113 {
3114 	return test_bit(__LINK_STATE_PRESENT, &dev->state);
3115 }
3116 
3117 void netif_device_detach(struct net_device *dev);
3118 
3119 void netif_device_attach(struct net_device *dev);
3120 
3121 /*
3122  * Network interface message level settings
3123  */
3124 
3125 enum {
3126 	NETIF_MSG_DRV		= 0x0001,
3127 	NETIF_MSG_PROBE		= 0x0002,
3128 	NETIF_MSG_LINK		= 0x0004,
3129 	NETIF_MSG_TIMER		= 0x0008,
3130 	NETIF_MSG_IFDOWN	= 0x0010,
3131 	NETIF_MSG_IFUP		= 0x0020,
3132 	NETIF_MSG_RX_ERR	= 0x0040,
3133 	NETIF_MSG_TX_ERR	= 0x0080,
3134 	NETIF_MSG_TX_QUEUED	= 0x0100,
3135 	NETIF_MSG_INTR		= 0x0200,
3136 	NETIF_MSG_TX_DONE	= 0x0400,
3137 	NETIF_MSG_RX_STATUS	= 0x0800,
3138 	NETIF_MSG_PKTDATA	= 0x1000,
3139 	NETIF_MSG_HW		= 0x2000,
3140 	NETIF_MSG_WOL		= 0x4000,
3141 };
3142 
3143 #define netif_msg_drv(p)	((p)->msg_enable & NETIF_MSG_DRV)
3144 #define netif_msg_probe(p)	((p)->msg_enable & NETIF_MSG_PROBE)
3145 #define netif_msg_link(p)	((p)->msg_enable & NETIF_MSG_LINK)
3146 #define netif_msg_timer(p)	((p)->msg_enable & NETIF_MSG_TIMER)
3147 #define netif_msg_ifdown(p)	((p)->msg_enable & NETIF_MSG_IFDOWN)
3148 #define netif_msg_ifup(p)	((p)->msg_enable & NETIF_MSG_IFUP)
3149 #define netif_msg_rx_err(p)	((p)->msg_enable & NETIF_MSG_RX_ERR)
3150 #define netif_msg_tx_err(p)	((p)->msg_enable & NETIF_MSG_TX_ERR)
3151 #define netif_msg_tx_queued(p)	((p)->msg_enable & NETIF_MSG_TX_QUEUED)
3152 #define netif_msg_intr(p)	((p)->msg_enable & NETIF_MSG_INTR)
3153 #define netif_msg_tx_done(p)	((p)->msg_enable & NETIF_MSG_TX_DONE)
3154 #define netif_msg_rx_status(p)	((p)->msg_enable & NETIF_MSG_RX_STATUS)
3155 #define netif_msg_pktdata(p)	((p)->msg_enable & NETIF_MSG_PKTDATA)
3156 #define netif_msg_hw(p)		((p)->msg_enable & NETIF_MSG_HW)
3157 #define netif_msg_wol(p)	((p)->msg_enable & NETIF_MSG_WOL)
3158 
3159 static inline u32 netif_msg_init(int debug_value, int default_msg_enable_bits)
3160 {
3161 	/* use default */
3162 	if (debug_value < 0 || debug_value >= (sizeof(u32) * 8))
3163 		return default_msg_enable_bits;
3164 	if (debug_value == 0)	/* no output */
3165 		return 0;
3166 	/* set low N bits */
3167 	return (1 << debug_value) - 1;
3168 }
3169 
3170 static inline void __netif_tx_lock(struct netdev_queue *txq, int cpu)
3171 {
3172 	spin_lock(&txq->_xmit_lock);
3173 	txq->xmit_lock_owner = cpu;
3174 }
3175 
3176 static inline void __netif_tx_lock_bh(struct netdev_queue *txq)
3177 {
3178 	spin_lock_bh(&txq->_xmit_lock);
3179 	txq->xmit_lock_owner = smp_processor_id();
3180 }
3181 
3182 static inline bool __netif_tx_trylock(struct netdev_queue *txq)
3183 {
3184 	bool ok = spin_trylock(&txq->_xmit_lock);
3185 	if (likely(ok))
3186 		txq->xmit_lock_owner = smp_processor_id();
3187 	return ok;
3188 }
3189 
3190 static inline void __netif_tx_unlock(struct netdev_queue *txq)
3191 {
3192 	txq->xmit_lock_owner = -1;
3193 	spin_unlock(&txq->_xmit_lock);
3194 }
3195 
3196 static inline void __netif_tx_unlock_bh(struct netdev_queue *txq)
3197 {
3198 	txq->xmit_lock_owner = -1;
3199 	spin_unlock_bh(&txq->_xmit_lock);
3200 }
3201 
3202 static inline void txq_trans_update(struct netdev_queue *txq)
3203 {
3204 	if (txq->xmit_lock_owner != -1)
3205 		txq->trans_start = jiffies;
3206 }
3207 
3208 /**
3209  *	netif_tx_lock - grab network device transmit lock
3210  *	@dev: network device
3211  *
3212  * Get network device transmit lock
3213  */
3214 static inline void netif_tx_lock(struct net_device *dev)
3215 {
3216 	unsigned int i;
3217 	int cpu;
3218 
3219 	spin_lock(&dev->tx_global_lock);
3220 	cpu = smp_processor_id();
3221 	for (i = 0; i < dev->num_tx_queues; i++) {
3222 		struct netdev_queue *txq = netdev_get_tx_queue(dev, i);
3223 
3224 		/* We are the only thread of execution doing a
3225 		 * freeze, but we have to grab the _xmit_lock in
3226 		 * order to synchronize with threads which are in
3227 		 * the ->hard_start_xmit() handler and already
3228 		 * checked the frozen bit.
3229 		 */
3230 		__netif_tx_lock(txq, cpu);
3231 		set_bit(__QUEUE_STATE_FROZEN, &txq->state);
3232 		__netif_tx_unlock(txq);
3233 	}
3234 }
3235 
3236 static inline void netif_tx_lock_bh(struct net_device *dev)
3237 {
3238 	local_bh_disable();
3239 	netif_tx_lock(dev);
3240 }
3241 
3242 static inline void netif_tx_unlock(struct net_device *dev)
3243 {
3244 	unsigned int i;
3245 
3246 	for (i = 0; i < dev->num_tx_queues; i++) {
3247 		struct netdev_queue *txq = netdev_get_tx_queue(dev, i);
3248 
3249 		/* No need to grab the _xmit_lock here.  If the
3250 		 * queue is not stopped for another reason, we
3251 		 * force a schedule.
3252 		 */
3253 		clear_bit(__QUEUE_STATE_FROZEN, &txq->state);
3254 		netif_schedule_queue(txq);
3255 	}
3256 	spin_unlock(&dev->tx_global_lock);
3257 }
3258 
3259 static inline void netif_tx_unlock_bh(struct net_device *dev)
3260 {
3261 	netif_tx_unlock(dev);
3262 	local_bh_enable();
3263 }
3264 
3265 #define HARD_TX_LOCK(dev, txq, cpu) {			\
3266 	if ((dev->features & NETIF_F_LLTX) == 0) {	\
3267 		__netif_tx_lock(txq, cpu);		\
3268 	}						\
3269 }
3270 
3271 #define HARD_TX_TRYLOCK(dev, txq)			\
3272 	(((dev->features & NETIF_F_LLTX) == 0) ?	\
3273 		__netif_tx_trylock(txq) :		\
3274 		true )
3275 
3276 #define HARD_TX_UNLOCK(dev, txq) {			\
3277 	if ((dev->features & NETIF_F_LLTX) == 0) {	\
3278 		__netif_tx_unlock(txq);			\
3279 	}						\
3280 }
3281 
3282 static inline void netif_tx_disable(struct net_device *dev)
3283 {
3284 	unsigned int i;
3285 	int cpu;
3286 
3287 	local_bh_disable();
3288 	cpu = smp_processor_id();
3289 	for (i = 0; i < dev->num_tx_queues; i++) {
3290 		struct netdev_queue *txq = netdev_get_tx_queue(dev, i);
3291 
3292 		__netif_tx_lock(txq, cpu);
3293 		netif_tx_stop_queue(txq);
3294 		__netif_tx_unlock(txq);
3295 	}
3296 	local_bh_enable();
3297 }
3298 
3299 static inline void netif_addr_lock(struct net_device *dev)
3300 {
3301 	spin_lock(&dev->addr_list_lock);
3302 }
3303 
3304 static inline void netif_addr_lock_nested(struct net_device *dev)
3305 {
3306 	int subclass = SINGLE_DEPTH_NESTING;
3307 
3308 	if (dev->netdev_ops->ndo_get_lock_subclass)
3309 		subclass = dev->netdev_ops->ndo_get_lock_subclass(dev);
3310 
3311 	spin_lock_nested(&dev->addr_list_lock, subclass);
3312 }
3313 
3314 static inline void netif_addr_lock_bh(struct net_device *dev)
3315 {
3316 	spin_lock_bh(&dev->addr_list_lock);
3317 }
3318 
3319 static inline void netif_addr_unlock(struct net_device *dev)
3320 {
3321 	spin_unlock(&dev->addr_list_lock);
3322 }
3323 
3324 static inline void netif_addr_unlock_bh(struct net_device *dev)
3325 {
3326 	spin_unlock_bh(&dev->addr_list_lock);
3327 }
3328 
3329 /*
3330  * dev_addrs walker. Should be used only for read access. Call with
3331  * rcu_read_lock held.
3332  */
3333 #define for_each_dev_addr(dev, ha) \
3334 		list_for_each_entry_rcu(ha, &dev->dev_addrs.list, list)
3335 
3336 /* These functions live elsewhere (drivers/net/net_init.c, but related) */
3337 
3338 void ether_setup(struct net_device *dev);
3339 
3340 /* Support for loadable net-drivers */
3341 struct net_device *alloc_netdev_mqs(int sizeof_priv, const char *name,
3342 				    unsigned char name_assign_type,
3343 				    void (*setup)(struct net_device *),
3344 				    unsigned int txqs, unsigned int rxqs);
3345 #define alloc_netdev(sizeof_priv, name, name_assign_type, setup) \
3346 	alloc_netdev_mqs(sizeof_priv, name, name_assign_type, setup, 1, 1)
3347 
3348 #define alloc_netdev_mq(sizeof_priv, name, name_assign_type, setup, count) \
3349 	alloc_netdev_mqs(sizeof_priv, name, name_assign_type, setup, count, \
3350 			 count)
3351 
3352 int register_netdev(struct net_device *dev);
3353 void unregister_netdev(struct net_device *dev);
3354 
3355 /* General hardware address lists handling functions */
3356 int __hw_addr_sync(struct netdev_hw_addr_list *to_list,
3357 		   struct netdev_hw_addr_list *from_list, int addr_len);
3358 void __hw_addr_unsync(struct netdev_hw_addr_list *to_list,
3359 		      struct netdev_hw_addr_list *from_list, int addr_len);
3360 int __hw_addr_sync_dev(struct netdev_hw_addr_list *list,
3361 		       struct net_device *dev,
3362 		       int (*sync)(struct net_device *, const unsigned char *),
3363 		       int (*unsync)(struct net_device *,
3364 				     const unsigned char *));
3365 void __hw_addr_unsync_dev(struct netdev_hw_addr_list *list,
3366 			  struct net_device *dev,
3367 			  int (*unsync)(struct net_device *,
3368 					const unsigned char *));
3369 void __hw_addr_init(struct netdev_hw_addr_list *list);
3370 
3371 /* Functions used for device addresses handling */
3372 int dev_addr_add(struct net_device *dev, const unsigned char *addr,
3373 		 unsigned char addr_type);
3374 int dev_addr_del(struct net_device *dev, const unsigned char *addr,
3375 		 unsigned char addr_type);
3376 void dev_addr_flush(struct net_device *dev);
3377 int dev_addr_init(struct net_device *dev);
3378 
3379 /* Functions used for unicast addresses handling */
3380 int dev_uc_add(struct net_device *dev, const unsigned char *addr);
3381 int dev_uc_add_excl(struct net_device *dev, const unsigned char *addr);
3382 int dev_uc_del(struct net_device *dev, const unsigned char *addr);
3383 int dev_uc_sync(struct net_device *to, struct net_device *from);
3384 int dev_uc_sync_multiple(struct net_device *to, struct net_device *from);
3385 void dev_uc_unsync(struct net_device *to, struct net_device *from);
3386 void dev_uc_flush(struct net_device *dev);
3387 void dev_uc_init(struct net_device *dev);
3388 
3389 /**
3390  *  __dev_uc_sync - Synchonize device's unicast list
3391  *  @dev:  device to sync
3392  *  @sync: function to call if address should be added
3393  *  @unsync: function to call if address should be removed
3394  *
3395  *  Add newly added addresses to the interface, and release
3396  *  addresses that have been deleted.
3397  **/
3398 static inline int __dev_uc_sync(struct net_device *dev,
3399 				int (*sync)(struct net_device *,
3400 					    const unsigned char *),
3401 				int (*unsync)(struct net_device *,
3402 					      const unsigned char *))
3403 {
3404 	return __hw_addr_sync_dev(&dev->uc, dev, sync, unsync);
3405 }
3406 
3407 /**
3408  *  __dev_uc_unsync - Remove synchronized addresses from device
3409  *  @dev:  device to sync
3410  *  @unsync: function to call if address should be removed
3411  *
3412  *  Remove all addresses that were added to the device by dev_uc_sync().
3413  **/
3414 static inline void __dev_uc_unsync(struct net_device *dev,
3415 				   int (*unsync)(struct net_device *,
3416 						 const unsigned char *))
3417 {
3418 	__hw_addr_unsync_dev(&dev->uc, dev, unsync);
3419 }
3420 
3421 /* Functions used for multicast addresses handling */
3422 int dev_mc_add(struct net_device *dev, const unsigned char *addr);
3423 int dev_mc_add_global(struct net_device *dev, const unsigned char *addr);
3424 int dev_mc_add_excl(struct net_device *dev, const unsigned char *addr);
3425 int dev_mc_del(struct net_device *dev, const unsigned char *addr);
3426 int dev_mc_del_global(struct net_device *dev, const unsigned char *addr);
3427 int dev_mc_sync(struct net_device *to, struct net_device *from);
3428 int dev_mc_sync_multiple(struct net_device *to, struct net_device *from);
3429 void dev_mc_unsync(struct net_device *to, struct net_device *from);
3430 void dev_mc_flush(struct net_device *dev);
3431 void dev_mc_init(struct net_device *dev);
3432 
3433 /**
3434  *  __dev_mc_sync - Synchonize device's multicast list
3435  *  @dev:  device to sync
3436  *  @sync: function to call if address should be added
3437  *  @unsync: function to call if address should be removed
3438  *
3439  *  Add newly added addresses to the interface, and release
3440  *  addresses that have been deleted.
3441  **/
3442 static inline int __dev_mc_sync(struct net_device *dev,
3443 				int (*sync)(struct net_device *,
3444 					    const unsigned char *),
3445 				int (*unsync)(struct net_device *,
3446 					      const unsigned char *))
3447 {
3448 	return __hw_addr_sync_dev(&dev->mc, dev, sync, unsync);
3449 }
3450 
3451 /**
3452  *  __dev_mc_unsync - Remove synchronized addresses from device
3453  *  @dev:  device to sync
3454  *  @unsync: function to call if address should be removed
3455  *
3456  *  Remove all addresses that were added to the device by dev_mc_sync().
3457  **/
3458 static inline void __dev_mc_unsync(struct net_device *dev,
3459 				   int (*unsync)(struct net_device *,
3460 						 const unsigned char *))
3461 {
3462 	__hw_addr_unsync_dev(&dev->mc, dev, unsync);
3463 }
3464 
3465 /* Functions used for secondary unicast and multicast support */
3466 void dev_set_rx_mode(struct net_device *dev);
3467 void __dev_set_rx_mode(struct net_device *dev);
3468 int dev_set_promiscuity(struct net_device *dev, int inc);
3469 int dev_set_allmulti(struct net_device *dev, int inc);
3470 void netdev_state_change(struct net_device *dev);
3471 void netdev_notify_peers(struct net_device *dev);
3472 void netdev_features_change(struct net_device *dev);
3473 /* Load a device via the kmod */
3474 void dev_load(struct net *net, const char *name);
3475 struct rtnl_link_stats64 *dev_get_stats(struct net_device *dev,
3476 					struct rtnl_link_stats64 *storage);
3477 void netdev_stats_to_stats64(struct rtnl_link_stats64 *stats64,
3478 			     const struct net_device_stats *netdev_stats);
3479 
3480 extern int		netdev_max_backlog;
3481 extern int		netdev_tstamp_prequeue;
3482 extern int		weight_p;
3483 extern int		bpf_jit_enable;
3484 
3485 bool netdev_has_upper_dev(struct net_device *dev, struct net_device *upper_dev);
3486 struct net_device *netdev_upper_get_next_dev_rcu(struct net_device *dev,
3487 						     struct list_head **iter);
3488 struct net_device *netdev_all_upper_get_next_dev_rcu(struct net_device *dev,
3489 						     struct list_head **iter);
3490 
3491 /* iterate through upper list, must be called under RCU read lock */
3492 #define netdev_for_each_upper_dev_rcu(dev, updev, iter) \
3493 	for (iter = &(dev)->adj_list.upper, \
3494 	     updev = netdev_upper_get_next_dev_rcu(dev, &(iter)); \
3495 	     updev; \
3496 	     updev = netdev_upper_get_next_dev_rcu(dev, &(iter)))
3497 
3498 /* iterate through upper list, must be called under RCU read lock */
3499 #define netdev_for_each_all_upper_dev_rcu(dev, updev, iter) \
3500 	for (iter = &(dev)->all_adj_list.upper, \
3501 	     updev = netdev_all_upper_get_next_dev_rcu(dev, &(iter)); \
3502 	     updev; \
3503 	     updev = netdev_all_upper_get_next_dev_rcu(dev, &(iter)))
3504 
3505 void *netdev_lower_get_next_private(struct net_device *dev,
3506 				    struct list_head **iter);
3507 void *netdev_lower_get_next_private_rcu(struct net_device *dev,
3508 					struct list_head **iter);
3509 
3510 #define netdev_for_each_lower_private(dev, priv, iter) \
3511 	for (iter = (dev)->adj_list.lower.next, \
3512 	     priv = netdev_lower_get_next_private(dev, &(iter)); \
3513 	     priv; \
3514 	     priv = netdev_lower_get_next_private(dev, &(iter)))
3515 
3516 #define netdev_for_each_lower_private_rcu(dev, priv, iter) \
3517 	for (iter = &(dev)->adj_list.lower, \
3518 	     priv = netdev_lower_get_next_private_rcu(dev, &(iter)); \
3519 	     priv; \
3520 	     priv = netdev_lower_get_next_private_rcu(dev, &(iter)))
3521 
3522 void *netdev_lower_get_next(struct net_device *dev,
3523 				struct list_head **iter);
3524 #define netdev_for_each_lower_dev(dev, ldev, iter) \
3525 	for (iter = &(dev)->adj_list.lower, \
3526 	     ldev = netdev_lower_get_next(dev, &(iter)); \
3527 	     ldev; \
3528 	     ldev = netdev_lower_get_next(dev, &(iter)))
3529 
3530 void *netdev_adjacent_get_private(struct list_head *adj_list);
3531 void *netdev_lower_get_first_private_rcu(struct net_device *dev);
3532 struct net_device *netdev_master_upper_dev_get(struct net_device *dev);
3533 struct net_device *netdev_master_upper_dev_get_rcu(struct net_device *dev);
3534 int netdev_upper_dev_link(struct net_device *dev, struct net_device *upper_dev);
3535 int netdev_master_upper_dev_link(struct net_device *dev,
3536 				 struct net_device *upper_dev);
3537 int netdev_master_upper_dev_link_private(struct net_device *dev,
3538 					 struct net_device *upper_dev,
3539 					 void *private);
3540 void netdev_upper_dev_unlink(struct net_device *dev,
3541 			     struct net_device *upper_dev);
3542 void netdev_adjacent_rename_links(struct net_device *dev, char *oldname);
3543 void *netdev_lower_dev_get_private(struct net_device *dev,
3544 				   struct net_device *lower_dev);
3545 
3546 /* RSS keys are 40 or 52 bytes long */
3547 #define NETDEV_RSS_KEY_LEN 52
3548 extern u8 netdev_rss_key[NETDEV_RSS_KEY_LEN];
3549 void netdev_rss_key_fill(void *buffer, size_t len);
3550 
3551 int dev_get_nest_level(struct net_device *dev,
3552 		       bool (*type_check)(struct net_device *dev));
3553 int skb_checksum_help(struct sk_buff *skb);
3554 struct sk_buff *__skb_gso_segment(struct sk_buff *skb,
3555 				  netdev_features_t features, bool tx_path);
3556 struct sk_buff *skb_mac_gso_segment(struct sk_buff *skb,
3557 				    netdev_features_t features);
3558 
3559 struct netdev_bonding_info {
3560 	ifslave	slave;
3561 	ifbond	master;
3562 };
3563 
3564 struct netdev_notifier_bonding_info {
3565 	struct netdev_notifier_info info; /* must be first */
3566 	struct netdev_bonding_info  bonding_info;
3567 };
3568 
3569 void netdev_bonding_info_change(struct net_device *dev,
3570 				struct netdev_bonding_info *bonding_info);
3571 
3572 static inline
3573 struct sk_buff *skb_gso_segment(struct sk_buff *skb, netdev_features_t features)
3574 {
3575 	return __skb_gso_segment(skb, features, true);
3576 }
3577 __be16 skb_network_protocol(struct sk_buff *skb, int *depth);
3578 
3579 static inline bool can_checksum_protocol(netdev_features_t features,
3580 					 __be16 protocol)
3581 {
3582 	return ((features & NETIF_F_GEN_CSUM) ||
3583 		((features & NETIF_F_V4_CSUM) &&
3584 		 protocol == htons(ETH_P_IP)) ||
3585 		((features & NETIF_F_V6_CSUM) &&
3586 		 protocol == htons(ETH_P_IPV6)) ||
3587 		((features & NETIF_F_FCOE_CRC) &&
3588 		 protocol == htons(ETH_P_FCOE)));
3589 }
3590 
3591 #ifdef CONFIG_BUG
3592 void netdev_rx_csum_fault(struct net_device *dev);
3593 #else
3594 static inline void netdev_rx_csum_fault(struct net_device *dev)
3595 {
3596 }
3597 #endif
3598 /* rx skb timestamps */
3599 void net_enable_timestamp(void);
3600 void net_disable_timestamp(void);
3601 
3602 #ifdef CONFIG_PROC_FS
3603 int __init dev_proc_init(void);
3604 #else
3605 #define dev_proc_init() 0
3606 #endif
3607 
3608 static inline netdev_tx_t __netdev_start_xmit(const struct net_device_ops *ops,
3609 					      struct sk_buff *skb, struct net_device *dev,
3610 					      bool more)
3611 {
3612 	skb->xmit_more = more ? 1 : 0;
3613 	return ops->ndo_start_xmit(skb, dev);
3614 }
3615 
3616 static inline netdev_tx_t netdev_start_xmit(struct sk_buff *skb, struct net_device *dev,
3617 					    struct netdev_queue *txq, bool more)
3618 {
3619 	const struct net_device_ops *ops = dev->netdev_ops;
3620 	int rc;
3621 
3622 	rc = __netdev_start_xmit(ops, skb, dev, more);
3623 	if (rc == NETDEV_TX_OK)
3624 		txq_trans_update(txq);
3625 
3626 	return rc;
3627 }
3628 
3629 int netdev_class_create_file_ns(struct class_attribute *class_attr,
3630 				const void *ns);
3631 void netdev_class_remove_file_ns(struct class_attribute *class_attr,
3632 				 const void *ns);
3633 
3634 static inline int netdev_class_create_file(struct class_attribute *class_attr)
3635 {
3636 	return netdev_class_create_file_ns(class_attr, NULL);
3637 }
3638 
3639 static inline void netdev_class_remove_file(struct class_attribute *class_attr)
3640 {
3641 	netdev_class_remove_file_ns(class_attr, NULL);
3642 }
3643 
3644 extern struct kobj_ns_type_operations net_ns_type_operations;
3645 
3646 const char *netdev_drivername(const struct net_device *dev);
3647 
3648 void linkwatch_run_queue(void);
3649 
3650 static inline netdev_features_t netdev_intersect_features(netdev_features_t f1,
3651 							  netdev_features_t f2)
3652 {
3653 	if (f1 & NETIF_F_GEN_CSUM)
3654 		f1 |= (NETIF_F_ALL_CSUM & ~NETIF_F_GEN_CSUM);
3655 	if (f2 & NETIF_F_GEN_CSUM)
3656 		f2 |= (NETIF_F_ALL_CSUM & ~NETIF_F_GEN_CSUM);
3657 	f1 &= f2;
3658 	if (f1 & NETIF_F_GEN_CSUM)
3659 		f1 &= ~(NETIF_F_ALL_CSUM & ~NETIF_F_GEN_CSUM);
3660 
3661 	return f1;
3662 }
3663 
3664 static inline netdev_features_t netdev_get_wanted_features(
3665 	struct net_device *dev)
3666 {
3667 	return (dev->features & ~dev->hw_features) | dev->wanted_features;
3668 }
3669 netdev_features_t netdev_increment_features(netdev_features_t all,
3670 	netdev_features_t one, netdev_features_t mask);
3671 
3672 /* Allow TSO being used on stacked device :
3673  * Performing the GSO segmentation before last device
3674  * is a performance improvement.
3675  */
3676 static inline netdev_features_t netdev_add_tso_features(netdev_features_t features,
3677 							netdev_features_t mask)
3678 {
3679 	return netdev_increment_features(features, NETIF_F_ALL_TSO, mask);
3680 }
3681 
3682 int __netdev_update_features(struct net_device *dev);
3683 void netdev_update_features(struct net_device *dev);
3684 void netdev_change_features(struct net_device *dev);
3685 
3686 void netif_stacked_transfer_operstate(const struct net_device *rootdev,
3687 					struct net_device *dev);
3688 
3689 netdev_features_t passthru_features_check(struct sk_buff *skb,
3690 					  struct net_device *dev,
3691 					  netdev_features_t features);
3692 netdev_features_t netif_skb_features(struct sk_buff *skb);
3693 
3694 static inline bool net_gso_ok(netdev_features_t features, int gso_type)
3695 {
3696 	netdev_features_t feature = gso_type << NETIF_F_GSO_SHIFT;
3697 
3698 	/* check flags correspondence */
3699 	BUILD_BUG_ON(SKB_GSO_TCPV4   != (NETIF_F_TSO >> NETIF_F_GSO_SHIFT));
3700 	BUILD_BUG_ON(SKB_GSO_UDP     != (NETIF_F_UFO >> NETIF_F_GSO_SHIFT));
3701 	BUILD_BUG_ON(SKB_GSO_DODGY   != (NETIF_F_GSO_ROBUST >> NETIF_F_GSO_SHIFT));
3702 	BUILD_BUG_ON(SKB_GSO_TCP_ECN != (NETIF_F_TSO_ECN >> NETIF_F_GSO_SHIFT));
3703 	BUILD_BUG_ON(SKB_GSO_TCPV6   != (NETIF_F_TSO6 >> NETIF_F_GSO_SHIFT));
3704 	BUILD_BUG_ON(SKB_GSO_FCOE    != (NETIF_F_FSO >> NETIF_F_GSO_SHIFT));
3705 	BUILD_BUG_ON(SKB_GSO_GRE     != (NETIF_F_GSO_GRE >> NETIF_F_GSO_SHIFT));
3706 	BUILD_BUG_ON(SKB_GSO_GRE_CSUM != (NETIF_F_GSO_GRE_CSUM >> NETIF_F_GSO_SHIFT));
3707 	BUILD_BUG_ON(SKB_GSO_IPIP    != (NETIF_F_GSO_IPIP >> NETIF_F_GSO_SHIFT));
3708 	BUILD_BUG_ON(SKB_GSO_SIT     != (NETIF_F_GSO_SIT >> NETIF_F_GSO_SHIFT));
3709 	BUILD_BUG_ON(SKB_GSO_UDP_TUNNEL != (NETIF_F_GSO_UDP_TUNNEL >> NETIF_F_GSO_SHIFT));
3710 	BUILD_BUG_ON(SKB_GSO_UDP_TUNNEL_CSUM != (NETIF_F_GSO_UDP_TUNNEL_CSUM >> NETIF_F_GSO_SHIFT));
3711 	BUILD_BUG_ON(SKB_GSO_TUNNEL_REMCSUM != (NETIF_F_GSO_TUNNEL_REMCSUM >> NETIF_F_GSO_SHIFT));
3712 
3713 	return (features & feature) == feature;
3714 }
3715 
3716 static inline bool skb_gso_ok(struct sk_buff *skb, netdev_features_t features)
3717 {
3718 	return net_gso_ok(features, skb_shinfo(skb)->gso_type) &&
3719 	       (!skb_has_frag_list(skb) || (features & NETIF_F_FRAGLIST));
3720 }
3721 
3722 static inline bool netif_needs_gso(struct sk_buff *skb,
3723 				   netdev_features_t features)
3724 {
3725 	return skb_is_gso(skb) && (!skb_gso_ok(skb, features) ||
3726 		unlikely((skb->ip_summed != CHECKSUM_PARTIAL) &&
3727 			 (skb->ip_summed != CHECKSUM_UNNECESSARY)));
3728 }
3729 
3730 static inline void netif_set_gso_max_size(struct net_device *dev,
3731 					  unsigned int size)
3732 {
3733 	dev->gso_max_size = size;
3734 }
3735 
3736 static inline void skb_gso_error_unwind(struct sk_buff *skb, __be16 protocol,
3737 					int pulled_hlen, u16 mac_offset,
3738 					int mac_len)
3739 {
3740 	skb->protocol = protocol;
3741 	skb->encapsulation = 1;
3742 	skb_push(skb, pulled_hlen);
3743 	skb_reset_transport_header(skb);
3744 	skb->mac_header = mac_offset;
3745 	skb->network_header = skb->mac_header + mac_len;
3746 	skb->mac_len = mac_len;
3747 }
3748 
3749 static inline bool netif_is_macvlan(struct net_device *dev)
3750 {
3751 	return dev->priv_flags & IFF_MACVLAN;
3752 }
3753 
3754 static inline bool netif_is_macvlan_port(struct net_device *dev)
3755 {
3756 	return dev->priv_flags & IFF_MACVLAN_PORT;
3757 }
3758 
3759 static inline bool netif_is_ipvlan(struct net_device *dev)
3760 {
3761 	return dev->priv_flags & IFF_IPVLAN_SLAVE;
3762 }
3763 
3764 static inline bool netif_is_ipvlan_port(struct net_device *dev)
3765 {
3766 	return dev->priv_flags & IFF_IPVLAN_MASTER;
3767 }
3768 
3769 static inline bool netif_is_bond_master(struct net_device *dev)
3770 {
3771 	return dev->flags & IFF_MASTER && dev->priv_flags & IFF_BONDING;
3772 }
3773 
3774 static inline bool netif_is_bond_slave(struct net_device *dev)
3775 {
3776 	return dev->flags & IFF_SLAVE && dev->priv_flags & IFF_BONDING;
3777 }
3778 
3779 static inline bool netif_supports_nofcs(struct net_device *dev)
3780 {
3781 	return dev->priv_flags & IFF_SUPP_NOFCS;
3782 }
3783 
3784 /* This device needs to keep skb dst for qdisc enqueue or ndo_start_xmit() */
3785 static inline void netif_keep_dst(struct net_device *dev)
3786 {
3787 	dev->priv_flags &= ~(IFF_XMIT_DST_RELEASE | IFF_XMIT_DST_RELEASE_PERM);
3788 }
3789 
3790 extern struct pernet_operations __net_initdata loopback_net_ops;
3791 
3792 /* Logging, debugging and troubleshooting/diagnostic helpers. */
3793 
3794 /* netdev_printk helpers, similar to dev_printk */
3795 
3796 static inline const char *netdev_name(const struct net_device *dev)
3797 {
3798 	if (!dev->name[0] || strchr(dev->name, '%'))
3799 		return "(unnamed net_device)";
3800 	return dev->name;
3801 }
3802 
3803 static inline const char *netdev_reg_state(const struct net_device *dev)
3804 {
3805 	switch (dev->reg_state) {
3806 	case NETREG_UNINITIALIZED: return " (uninitialized)";
3807 	case NETREG_REGISTERED: return "";
3808 	case NETREG_UNREGISTERING: return " (unregistering)";
3809 	case NETREG_UNREGISTERED: return " (unregistered)";
3810 	case NETREG_RELEASED: return " (released)";
3811 	case NETREG_DUMMY: return " (dummy)";
3812 	}
3813 
3814 	WARN_ONCE(1, "%s: unknown reg_state %d\n", dev->name, dev->reg_state);
3815 	return " (unknown)";
3816 }
3817 
3818 __printf(3, 4)
3819 void netdev_printk(const char *level, const struct net_device *dev,
3820 		   const char *format, ...);
3821 __printf(2, 3)
3822 void netdev_emerg(const struct net_device *dev, const char *format, ...);
3823 __printf(2, 3)
3824 void netdev_alert(const struct net_device *dev, const char *format, ...);
3825 __printf(2, 3)
3826 void netdev_crit(const struct net_device *dev, const char *format, ...);
3827 __printf(2, 3)
3828 void netdev_err(const struct net_device *dev, const char *format, ...);
3829 __printf(2, 3)
3830 void netdev_warn(const struct net_device *dev, const char *format, ...);
3831 __printf(2, 3)
3832 void netdev_notice(const struct net_device *dev, const char *format, ...);
3833 __printf(2, 3)
3834 void netdev_info(const struct net_device *dev, const char *format, ...);
3835 
3836 #define MODULE_ALIAS_NETDEV(device) \
3837 	MODULE_ALIAS("netdev-" device)
3838 
3839 #if defined(CONFIG_DYNAMIC_DEBUG)
3840 #define netdev_dbg(__dev, format, args...)			\
3841 do {								\
3842 	dynamic_netdev_dbg(__dev, format, ##args);		\
3843 } while (0)
3844 #elif defined(DEBUG)
3845 #define netdev_dbg(__dev, format, args...)			\
3846 	netdev_printk(KERN_DEBUG, __dev, format, ##args)
3847 #else
3848 #define netdev_dbg(__dev, format, args...)			\
3849 ({								\
3850 	if (0)							\
3851 		netdev_printk(KERN_DEBUG, __dev, format, ##args); \
3852 })
3853 #endif
3854 
3855 #if defined(VERBOSE_DEBUG)
3856 #define netdev_vdbg	netdev_dbg
3857 #else
3858 
3859 #define netdev_vdbg(dev, format, args...)			\
3860 ({								\
3861 	if (0)							\
3862 		netdev_printk(KERN_DEBUG, dev, format, ##args);	\
3863 	0;							\
3864 })
3865 #endif
3866 
3867 /*
3868  * netdev_WARN() acts like dev_printk(), but with the key difference
3869  * of using a WARN/WARN_ON to get the message out, including the
3870  * file/line information and a backtrace.
3871  */
3872 #define netdev_WARN(dev, format, args...)			\
3873 	WARN(1, "netdevice: %s%s\n" format, netdev_name(dev),	\
3874 	     netdev_reg_state(dev), ##args)
3875 
3876 /* netif printk helpers, similar to netdev_printk */
3877 
3878 #define netif_printk(priv, type, level, dev, fmt, args...)	\
3879 do {					  			\
3880 	if (netif_msg_##type(priv))				\
3881 		netdev_printk(level, (dev), fmt, ##args);	\
3882 } while (0)
3883 
3884 #define netif_level(level, priv, type, dev, fmt, args...)	\
3885 do {								\
3886 	if (netif_msg_##type(priv))				\
3887 		netdev_##level(dev, fmt, ##args);		\
3888 } while (0)
3889 
3890 #define netif_emerg(priv, type, dev, fmt, args...)		\
3891 	netif_level(emerg, priv, type, dev, fmt, ##args)
3892 #define netif_alert(priv, type, dev, fmt, args...)		\
3893 	netif_level(alert, priv, type, dev, fmt, ##args)
3894 #define netif_crit(priv, type, dev, fmt, args...)		\
3895 	netif_level(crit, priv, type, dev, fmt, ##args)
3896 #define netif_err(priv, type, dev, fmt, args...)		\
3897 	netif_level(err, priv, type, dev, fmt, ##args)
3898 #define netif_warn(priv, type, dev, fmt, args...)		\
3899 	netif_level(warn, priv, type, dev, fmt, ##args)
3900 #define netif_notice(priv, type, dev, fmt, args...)		\
3901 	netif_level(notice, priv, type, dev, fmt, ##args)
3902 #define netif_info(priv, type, dev, fmt, args...)		\
3903 	netif_level(info, priv, type, dev, fmt, ##args)
3904 
3905 #if defined(CONFIG_DYNAMIC_DEBUG)
3906 #define netif_dbg(priv, type, netdev, format, args...)		\
3907 do {								\
3908 	if (netif_msg_##type(priv))				\
3909 		dynamic_netdev_dbg(netdev, format, ##args);	\
3910 } while (0)
3911 #elif defined(DEBUG)
3912 #define netif_dbg(priv, type, dev, format, args...)		\
3913 	netif_printk(priv, type, KERN_DEBUG, dev, format, ##args)
3914 #else
3915 #define netif_dbg(priv, type, dev, format, args...)			\
3916 ({									\
3917 	if (0)								\
3918 		netif_printk(priv, type, KERN_DEBUG, dev, format, ##args); \
3919 	0;								\
3920 })
3921 #endif
3922 
3923 #if defined(VERBOSE_DEBUG)
3924 #define netif_vdbg	netif_dbg
3925 #else
3926 #define netif_vdbg(priv, type, dev, format, args...)		\
3927 ({								\
3928 	if (0)							\
3929 		netif_printk(priv, type, KERN_DEBUG, dev, format, ##args); \
3930 	0;							\
3931 })
3932 #endif
3933 
3934 /*
3935  *	The list of packet types we will receive (as opposed to discard)
3936  *	and the routines to invoke.
3937  *
3938  *	Why 16. Because with 16 the only overlap we get on a hash of the
3939  *	low nibble of the protocol value is RARP/SNAP/X.25.
3940  *
3941  *      NOTE:  That is no longer true with the addition of VLAN tags.  Not
3942  *             sure which should go first, but I bet it won't make much
3943  *             difference if we are running VLANs.  The good news is that
3944  *             this protocol won't be in the list unless compiled in, so
3945  *             the average user (w/out VLANs) will not be adversely affected.
3946  *             --BLG
3947  *
3948  *		0800	IP
3949  *		8100    802.1Q VLAN
3950  *		0001	802.3
3951  *		0002	AX.25
3952  *		0004	802.2
3953  *		8035	RARP
3954  *		0005	SNAP
3955  *		0805	X.25
3956  *		0806	ARP
3957  *		8137	IPX
3958  *		0009	Localtalk
3959  *		86DD	IPv6
3960  */
3961 #define PTYPE_HASH_SIZE	(16)
3962 #define PTYPE_HASH_MASK	(PTYPE_HASH_SIZE - 1)
3963 
3964 #endif	/* _LINUX_NETDEVICE_H */
3965