xref: /openbmc/linux/net/packet/af_packet.c (revision bdeeed09)
1 /*
2  * INET		An implementation of the TCP/IP protocol suite for the LINUX
3  *		operating system.  INET is implemented using the  BSD Socket
4  *		interface as the means of communication with the user level.
5  *
6  *		PACKET - implements raw packet sockets.
7  *
8  * Authors:	Ross Biro
9  *		Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
10  *		Alan Cox, <gw4pts@gw4pts.ampr.org>
11  *
12  * Fixes:
13  *		Alan Cox	:	verify_area() now used correctly
14  *		Alan Cox	:	new skbuff lists, look ma no backlogs!
15  *		Alan Cox	:	tidied skbuff lists.
16  *		Alan Cox	:	Now uses generic datagram routines I
17  *					added. Also fixed the peek/read crash
18  *					from all old Linux datagram code.
19  *		Alan Cox	:	Uses the improved datagram code.
20  *		Alan Cox	:	Added NULL's for socket options.
21  *		Alan Cox	:	Re-commented the code.
22  *		Alan Cox	:	Use new kernel side addressing
23  *		Rob Janssen	:	Correct MTU usage.
24  *		Dave Platt	:	Counter leaks caused by incorrect
25  *					interrupt locking and some slightly
26  *					dubious gcc output. Can you read
27  *					compiler: it said _VOLATILE_
28  *	Richard Kooijman	:	Timestamp fixes.
29  *		Alan Cox	:	New buffers. Use sk->mac.raw.
30  *		Alan Cox	:	sendmsg/recvmsg support.
31  *		Alan Cox	:	Protocol setting support
32  *	Alexey Kuznetsov	:	Untied from IPv4 stack.
33  *	Cyrus Durgin		:	Fixed kerneld for kmod.
34  *	Michal Ostrowski        :       Module initialization cleanup.
35  *         Ulises Alonso        :       Frame number limit removal and
36  *                                      packet_set_ring memory leak.
37  *		Eric Biederman	:	Allow for > 8 byte hardware addresses.
38  *					The convention is that longer addresses
39  *					will simply extend the hardware address
40  *					byte arrays at the end of sockaddr_ll
41  *					and packet_mreq.
42  *		Johann Baudy	:	Added TX RING.
43  *		Chetan Loke	:	Implemented TPACKET_V3 block abstraction
44  *					layer.
45  *					Copyright (C) 2011, <lokec@ccs.neu.edu>
46  *
47  *
48  *		This program is free software; you can redistribute it and/or
49  *		modify it under the terms of the GNU General Public License
50  *		as published by the Free Software Foundation; either version
51  *		2 of the License, or (at your option) any later version.
52  *
53  */
54 
55 #include <linux/types.h>
56 #include <linux/mm.h>
57 #include <linux/capability.h>
58 #include <linux/fcntl.h>
59 #include <linux/socket.h>
60 #include <linux/in.h>
61 #include <linux/inet.h>
62 #include <linux/netdevice.h>
63 #include <linux/if_packet.h>
64 #include <linux/wireless.h>
65 #include <linux/kernel.h>
66 #include <linux/kmod.h>
67 #include <linux/slab.h>
68 #include <linux/vmalloc.h>
69 #include <net/net_namespace.h>
70 #include <net/ip.h>
71 #include <net/protocol.h>
72 #include <linux/skbuff.h>
73 #include <net/sock.h>
74 #include <linux/errno.h>
75 #include <linux/timer.h>
76 #include <linux/uaccess.h>
77 #include <asm/ioctls.h>
78 #include <asm/page.h>
79 #include <asm/cacheflush.h>
80 #include <asm/io.h>
81 #include <linux/proc_fs.h>
82 #include <linux/seq_file.h>
83 #include <linux/poll.h>
84 #include <linux/module.h>
85 #include <linux/init.h>
86 #include <linux/mutex.h>
87 #include <linux/if_vlan.h>
88 #include <linux/virtio_net.h>
89 #include <linux/errqueue.h>
90 #include <linux/net_tstamp.h>
91 #include <linux/percpu.h>
92 #ifdef CONFIG_INET
93 #include <net/inet_common.h>
94 #endif
95 #include <linux/bpf.h>
96 #include <net/compat.h>
97 
98 #include "internal.h"
99 
100 /*
101    Assumptions:
102    - if device has no dev->hard_header routine, it adds and removes ll header
103      inside itself. In this case ll header is invisible outside of device,
104      but higher levels still should reserve dev->hard_header_len.
105      Some devices are enough clever to reallocate skb, when header
106      will not fit to reserved space (tunnel), another ones are silly
107      (PPP).
108    - packet socket receives packets with pulled ll header,
109      so that SOCK_RAW should push it back.
110 
111 On receive:
112 -----------
113 
114 Incoming, dev->hard_header!=NULL
115    mac_header -> ll header
116    data       -> data
117 
118 Outgoing, dev->hard_header!=NULL
119    mac_header -> ll header
120    data       -> ll header
121 
122 Incoming, dev->hard_header==NULL
123    mac_header -> UNKNOWN position. It is very likely, that it points to ll
124 		 header.  PPP makes it, that is wrong, because introduce
125 		 assymetry between rx and tx paths.
126    data       -> data
127 
128 Outgoing, dev->hard_header==NULL
129    mac_header -> data. ll header is still not built!
130    data       -> data
131 
132 Resume
133   If dev->hard_header==NULL we are unlikely to restore sensible ll header.
134 
135 
136 On transmit:
137 ------------
138 
139 dev->hard_header != NULL
140    mac_header -> ll header
141    data       -> ll header
142 
143 dev->hard_header == NULL (ll header is added by device, we cannot control it)
144    mac_header -> data
145    data       -> data
146 
147    We should set nh.raw on output to correct posistion,
148    packet classifier depends on it.
149  */
150 
151 /* Private packet socket structures. */
152 
153 /* identical to struct packet_mreq except it has
154  * a longer address field.
155  */
156 struct packet_mreq_max {
157 	int		mr_ifindex;
158 	unsigned short	mr_type;
159 	unsigned short	mr_alen;
160 	unsigned char	mr_address[MAX_ADDR_LEN];
161 };
162 
163 union tpacket_uhdr {
164 	struct tpacket_hdr  *h1;
165 	struct tpacket2_hdr *h2;
166 	struct tpacket3_hdr *h3;
167 	void *raw;
168 };
169 
170 static int packet_set_ring(struct sock *sk, union tpacket_req_u *req_u,
171 		int closing, int tx_ring);
172 
173 #define V3_ALIGNMENT	(8)
174 
175 #define BLK_HDR_LEN	(ALIGN(sizeof(struct tpacket_block_desc), V3_ALIGNMENT))
176 
177 #define BLK_PLUS_PRIV(sz_of_priv) \
178 	(BLK_HDR_LEN + ALIGN((sz_of_priv), V3_ALIGNMENT))
179 
180 #define BLOCK_STATUS(x)	((x)->hdr.bh1.block_status)
181 #define BLOCK_NUM_PKTS(x)	((x)->hdr.bh1.num_pkts)
182 #define BLOCK_O2FP(x)		((x)->hdr.bh1.offset_to_first_pkt)
183 #define BLOCK_LEN(x)		((x)->hdr.bh1.blk_len)
184 #define BLOCK_SNUM(x)		((x)->hdr.bh1.seq_num)
185 #define BLOCK_O2PRIV(x)	((x)->offset_to_priv)
186 #define BLOCK_PRIV(x)		((void *)((char *)(x) + BLOCK_O2PRIV(x)))
187 
188 struct packet_sock;
189 static int tpacket_rcv(struct sk_buff *skb, struct net_device *dev,
190 		       struct packet_type *pt, struct net_device *orig_dev);
191 
192 static void *packet_previous_frame(struct packet_sock *po,
193 		struct packet_ring_buffer *rb,
194 		int status);
195 static void packet_increment_head(struct packet_ring_buffer *buff);
196 static int prb_curr_blk_in_use(struct tpacket_block_desc *);
197 static void *prb_dispatch_next_block(struct tpacket_kbdq_core *,
198 			struct packet_sock *);
199 static void prb_retire_current_block(struct tpacket_kbdq_core *,
200 		struct packet_sock *, unsigned int status);
201 static int prb_queue_frozen(struct tpacket_kbdq_core *);
202 static void prb_open_block(struct tpacket_kbdq_core *,
203 		struct tpacket_block_desc *);
204 static void prb_retire_rx_blk_timer_expired(struct timer_list *);
205 static void _prb_refresh_rx_retire_blk_timer(struct tpacket_kbdq_core *);
206 static void prb_fill_rxhash(struct tpacket_kbdq_core *, struct tpacket3_hdr *);
207 static void prb_clear_rxhash(struct tpacket_kbdq_core *,
208 		struct tpacket3_hdr *);
209 static void prb_fill_vlan_info(struct tpacket_kbdq_core *,
210 		struct tpacket3_hdr *);
211 static void packet_flush_mclist(struct sock *sk);
212 static void packet_pick_tx_queue(struct net_device *dev, struct sk_buff *skb);
213 
214 struct packet_skb_cb {
215 	union {
216 		struct sockaddr_pkt pkt;
217 		union {
218 			/* Trick: alias skb original length with
219 			 * ll.sll_family and ll.protocol in order
220 			 * to save room.
221 			 */
222 			unsigned int origlen;
223 			struct sockaddr_ll ll;
224 		};
225 	} sa;
226 };
227 
228 #define vio_le() virtio_legacy_is_little_endian()
229 
230 #define PACKET_SKB_CB(__skb)	((struct packet_skb_cb *)((__skb)->cb))
231 
232 #define GET_PBDQC_FROM_RB(x)	((struct tpacket_kbdq_core *)(&(x)->prb_bdqc))
233 #define GET_PBLOCK_DESC(x, bid)	\
234 	((struct tpacket_block_desc *)((x)->pkbdq[(bid)].buffer))
235 #define GET_CURR_PBLOCK_DESC_FROM_CORE(x)	\
236 	((struct tpacket_block_desc *)((x)->pkbdq[(x)->kactive_blk_num].buffer))
237 #define GET_NEXT_PRB_BLK_NUM(x) \
238 	(((x)->kactive_blk_num < ((x)->knum_blocks-1)) ? \
239 	((x)->kactive_blk_num+1) : 0)
240 
241 static void __fanout_unlink(struct sock *sk, struct packet_sock *po);
242 static void __fanout_link(struct sock *sk, struct packet_sock *po);
243 
244 static int packet_direct_xmit(struct sk_buff *skb)
245 {
246 	struct net_device *dev = skb->dev;
247 	struct sk_buff *orig_skb = skb;
248 	struct netdev_queue *txq;
249 	int ret = NETDEV_TX_BUSY;
250 	bool again = false;
251 
252 	if (unlikely(!netif_running(dev) ||
253 		     !netif_carrier_ok(dev)))
254 		goto drop;
255 
256 	skb = validate_xmit_skb_list(skb, dev, &again);
257 	if (skb != orig_skb)
258 		goto drop;
259 
260 	packet_pick_tx_queue(dev, skb);
261 	txq = skb_get_tx_queue(dev, skb);
262 
263 	local_bh_disable();
264 
265 	HARD_TX_LOCK(dev, txq, smp_processor_id());
266 	if (!netif_xmit_frozen_or_drv_stopped(txq))
267 		ret = netdev_start_xmit(skb, dev, txq, false);
268 	HARD_TX_UNLOCK(dev, txq);
269 
270 	local_bh_enable();
271 
272 	if (!dev_xmit_complete(ret))
273 		kfree_skb(skb);
274 
275 	return ret;
276 drop:
277 	atomic_long_inc(&dev->tx_dropped);
278 	kfree_skb_list(skb);
279 	return NET_XMIT_DROP;
280 }
281 
282 static struct net_device *packet_cached_dev_get(struct packet_sock *po)
283 {
284 	struct net_device *dev;
285 
286 	rcu_read_lock();
287 	dev = rcu_dereference(po->cached_dev);
288 	if (likely(dev))
289 		dev_hold(dev);
290 	rcu_read_unlock();
291 
292 	return dev;
293 }
294 
295 static void packet_cached_dev_assign(struct packet_sock *po,
296 				     struct net_device *dev)
297 {
298 	rcu_assign_pointer(po->cached_dev, dev);
299 }
300 
301 static void packet_cached_dev_reset(struct packet_sock *po)
302 {
303 	RCU_INIT_POINTER(po->cached_dev, NULL);
304 }
305 
306 static bool packet_use_direct_xmit(const struct packet_sock *po)
307 {
308 	return po->xmit == packet_direct_xmit;
309 }
310 
311 static u16 __packet_pick_tx_queue(struct net_device *dev, struct sk_buff *skb)
312 {
313 	return (u16) raw_smp_processor_id() % dev->real_num_tx_queues;
314 }
315 
316 static void packet_pick_tx_queue(struct net_device *dev, struct sk_buff *skb)
317 {
318 	const struct net_device_ops *ops = dev->netdev_ops;
319 	u16 queue_index;
320 
321 	if (ops->ndo_select_queue) {
322 		queue_index = ops->ndo_select_queue(dev, skb, NULL,
323 						    __packet_pick_tx_queue);
324 		queue_index = netdev_cap_txqueue(dev, queue_index);
325 	} else {
326 		queue_index = __packet_pick_tx_queue(dev, skb);
327 	}
328 
329 	skb_set_queue_mapping(skb, queue_index);
330 }
331 
332 /* __register_prot_hook must be invoked through register_prot_hook
333  * or from a context in which asynchronous accesses to the packet
334  * socket is not possible (packet_create()).
335  */
336 static void __register_prot_hook(struct sock *sk)
337 {
338 	struct packet_sock *po = pkt_sk(sk);
339 
340 	if (!po->running) {
341 		if (po->fanout)
342 			__fanout_link(sk, po);
343 		else
344 			dev_add_pack(&po->prot_hook);
345 
346 		sock_hold(sk);
347 		po->running = 1;
348 	}
349 }
350 
351 static void register_prot_hook(struct sock *sk)
352 {
353 	lockdep_assert_held_once(&pkt_sk(sk)->bind_lock);
354 	__register_prot_hook(sk);
355 }
356 
357 /* If the sync parameter is true, we will temporarily drop
358  * the po->bind_lock and do a synchronize_net to make sure no
359  * asynchronous packet processing paths still refer to the elements
360  * of po->prot_hook.  If the sync parameter is false, it is the
361  * callers responsibility to take care of this.
362  */
363 static void __unregister_prot_hook(struct sock *sk, bool sync)
364 {
365 	struct packet_sock *po = pkt_sk(sk);
366 
367 	lockdep_assert_held_once(&po->bind_lock);
368 
369 	po->running = 0;
370 
371 	if (po->fanout)
372 		__fanout_unlink(sk, po);
373 	else
374 		__dev_remove_pack(&po->prot_hook);
375 
376 	__sock_put(sk);
377 
378 	if (sync) {
379 		spin_unlock(&po->bind_lock);
380 		synchronize_net();
381 		spin_lock(&po->bind_lock);
382 	}
383 }
384 
385 static void unregister_prot_hook(struct sock *sk, bool sync)
386 {
387 	struct packet_sock *po = pkt_sk(sk);
388 
389 	if (po->running)
390 		__unregister_prot_hook(sk, sync);
391 }
392 
393 static inline struct page * __pure pgv_to_page(void *addr)
394 {
395 	if (is_vmalloc_addr(addr))
396 		return vmalloc_to_page(addr);
397 	return virt_to_page(addr);
398 }
399 
400 static void __packet_set_status(struct packet_sock *po, void *frame, int status)
401 {
402 	union tpacket_uhdr h;
403 
404 	h.raw = frame;
405 	switch (po->tp_version) {
406 	case TPACKET_V1:
407 		h.h1->tp_status = status;
408 		flush_dcache_page(pgv_to_page(&h.h1->tp_status));
409 		break;
410 	case TPACKET_V2:
411 		h.h2->tp_status = status;
412 		flush_dcache_page(pgv_to_page(&h.h2->tp_status));
413 		break;
414 	case TPACKET_V3:
415 		h.h3->tp_status = status;
416 		flush_dcache_page(pgv_to_page(&h.h3->tp_status));
417 		break;
418 	default:
419 		WARN(1, "TPACKET version not supported.\n");
420 		BUG();
421 	}
422 
423 	smp_wmb();
424 }
425 
426 static int __packet_get_status(struct packet_sock *po, void *frame)
427 {
428 	union tpacket_uhdr h;
429 
430 	smp_rmb();
431 
432 	h.raw = frame;
433 	switch (po->tp_version) {
434 	case TPACKET_V1:
435 		flush_dcache_page(pgv_to_page(&h.h1->tp_status));
436 		return h.h1->tp_status;
437 	case TPACKET_V2:
438 		flush_dcache_page(pgv_to_page(&h.h2->tp_status));
439 		return h.h2->tp_status;
440 	case TPACKET_V3:
441 		flush_dcache_page(pgv_to_page(&h.h3->tp_status));
442 		return h.h3->tp_status;
443 	default:
444 		WARN(1, "TPACKET version not supported.\n");
445 		BUG();
446 		return 0;
447 	}
448 }
449 
450 static __u32 tpacket_get_timestamp(struct sk_buff *skb, struct timespec *ts,
451 				   unsigned int flags)
452 {
453 	struct skb_shared_hwtstamps *shhwtstamps = skb_hwtstamps(skb);
454 
455 	if (shhwtstamps &&
456 	    (flags & SOF_TIMESTAMPING_RAW_HARDWARE) &&
457 	    ktime_to_timespec_cond(shhwtstamps->hwtstamp, ts))
458 		return TP_STATUS_TS_RAW_HARDWARE;
459 
460 	if (ktime_to_timespec_cond(skb->tstamp, ts))
461 		return TP_STATUS_TS_SOFTWARE;
462 
463 	return 0;
464 }
465 
466 static __u32 __packet_set_timestamp(struct packet_sock *po, void *frame,
467 				    struct sk_buff *skb)
468 {
469 	union tpacket_uhdr h;
470 	struct timespec ts;
471 	__u32 ts_status;
472 
473 	if (!(ts_status = tpacket_get_timestamp(skb, &ts, po->tp_tstamp)))
474 		return 0;
475 
476 	h.raw = frame;
477 	switch (po->tp_version) {
478 	case TPACKET_V1:
479 		h.h1->tp_sec = ts.tv_sec;
480 		h.h1->tp_usec = ts.tv_nsec / NSEC_PER_USEC;
481 		break;
482 	case TPACKET_V2:
483 		h.h2->tp_sec = ts.tv_sec;
484 		h.h2->tp_nsec = ts.tv_nsec;
485 		break;
486 	case TPACKET_V3:
487 		h.h3->tp_sec = ts.tv_sec;
488 		h.h3->tp_nsec = ts.tv_nsec;
489 		break;
490 	default:
491 		WARN(1, "TPACKET version not supported.\n");
492 		BUG();
493 	}
494 
495 	/* one flush is safe, as both fields always lie on the same cacheline */
496 	flush_dcache_page(pgv_to_page(&h.h1->tp_sec));
497 	smp_wmb();
498 
499 	return ts_status;
500 }
501 
502 static void *packet_lookup_frame(struct packet_sock *po,
503 		struct packet_ring_buffer *rb,
504 		unsigned int position,
505 		int status)
506 {
507 	unsigned int pg_vec_pos, frame_offset;
508 	union tpacket_uhdr h;
509 
510 	pg_vec_pos = position / rb->frames_per_block;
511 	frame_offset = position % rb->frames_per_block;
512 
513 	h.raw = rb->pg_vec[pg_vec_pos].buffer +
514 		(frame_offset * rb->frame_size);
515 
516 	if (status != __packet_get_status(po, h.raw))
517 		return NULL;
518 
519 	return h.raw;
520 }
521 
522 static void *packet_current_frame(struct packet_sock *po,
523 		struct packet_ring_buffer *rb,
524 		int status)
525 {
526 	return packet_lookup_frame(po, rb, rb->head, status);
527 }
528 
529 static void prb_del_retire_blk_timer(struct tpacket_kbdq_core *pkc)
530 {
531 	del_timer_sync(&pkc->retire_blk_timer);
532 }
533 
534 static void prb_shutdown_retire_blk_timer(struct packet_sock *po,
535 		struct sk_buff_head *rb_queue)
536 {
537 	struct tpacket_kbdq_core *pkc;
538 
539 	pkc = GET_PBDQC_FROM_RB(&po->rx_ring);
540 
541 	spin_lock_bh(&rb_queue->lock);
542 	pkc->delete_blk_timer = 1;
543 	spin_unlock_bh(&rb_queue->lock);
544 
545 	prb_del_retire_blk_timer(pkc);
546 }
547 
548 static void prb_setup_retire_blk_timer(struct packet_sock *po)
549 {
550 	struct tpacket_kbdq_core *pkc;
551 
552 	pkc = GET_PBDQC_FROM_RB(&po->rx_ring);
553 	timer_setup(&pkc->retire_blk_timer, prb_retire_rx_blk_timer_expired,
554 		    0);
555 	pkc->retire_blk_timer.expires = jiffies;
556 }
557 
558 static int prb_calc_retire_blk_tmo(struct packet_sock *po,
559 				int blk_size_in_bytes)
560 {
561 	struct net_device *dev;
562 	unsigned int mbits = 0, msec = 0, div = 0, tmo = 0;
563 	struct ethtool_link_ksettings ecmd;
564 	int err;
565 
566 	rtnl_lock();
567 	dev = __dev_get_by_index(sock_net(&po->sk), po->ifindex);
568 	if (unlikely(!dev)) {
569 		rtnl_unlock();
570 		return DEFAULT_PRB_RETIRE_TOV;
571 	}
572 	err = __ethtool_get_link_ksettings(dev, &ecmd);
573 	rtnl_unlock();
574 	if (!err) {
575 		/*
576 		 * If the link speed is so slow you don't really
577 		 * need to worry about perf anyways
578 		 */
579 		if (ecmd.base.speed < SPEED_1000 ||
580 		    ecmd.base.speed == SPEED_UNKNOWN) {
581 			return DEFAULT_PRB_RETIRE_TOV;
582 		} else {
583 			msec = 1;
584 			div = ecmd.base.speed / 1000;
585 		}
586 	}
587 
588 	mbits = (blk_size_in_bytes * 8) / (1024 * 1024);
589 
590 	if (div)
591 		mbits /= div;
592 
593 	tmo = mbits * msec;
594 
595 	if (div)
596 		return tmo+1;
597 	return tmo;
598 }
599 
600 static void prb_init_ft_ops(struct tpacket_kbdq_core *p1,
601 			union tpacket_req_u *req_u)
602 {
603 	p1->feature_req_word = req_u->req3.tp_feature_req_word;
604 }
605 
606 static void init_prb_bdqc(struct packet_sock *po,
607 			struct packet_ring_buffer *rb,
608 			struct pgv *pg_vec,
609 			union tpacket_req_u *req_u)
610 {
611 	struct tpacket_kbdq_core *p1 = GET_PBDQC_FROM_RB(rb);
612 	struct tpacket_block_desc *pbd;
613 
614 	memset(p1, 0x0, sizeof(*p1));
615 
616 	p1->knxt_seq_num = 1;
617 	p1->pkbdq = pg_vec;
618 	pbd = (struct tpacket_block_desc *)pg_vec[0].buffer;
619 	p1->pkblk_start	= pg_vec[0].buffer;
620 	p1->kblk_size = req_u->req3.tp_block_size;
621 	p1->knum_blocks	= req_u->req3.tp_block_nr;
622 	p1->hdrlen = po->tp_hdrlen;
623 	p1->version = po->tp_version;
624 	p1->last_kactive_blk_num = 0;
625 	po->stats.stats3.tp_freeze_q_cnt = 0;
626 	if (req_u->req3.tp_retire_blk_tov)
627 		p1->retire_blk_tov = req_u->req3.tp_retire_blk_tov;
628 	else
629 		p1->retire_blk_tov = prb_calc_retire_blk_tmo(po,
630 						req_u->req3.tp_block_size);
631 	p1->tov_in_jiffies = msecs_to_jiffies(p1->retire_blk_tov);
632 	p1->blk_sizeof_priv = req_u->req3.tp_sizeof_priv;
633 
634 	p1->max_frame_len = p1->kblk_size - BLK_PLUS_PRIV(p1->blk_sizeof_priv);
635 	prb_init_ft_ops(p1, req_u);
636 	prb_setup_retire_blk_timer(po);
637 	prb_open_block(p1, pbd);
638 }
639 
640 /*  Do NOT update the last_blk_num first.
641  *  Assumes sk_buff_head lock is held.
642  */
643 static void _prb_refresh_rx_retire_blk_timer(struct tpacket_kbdq_core *pkc)
644 {
645 	mod_timer(&pkc->retire_blk_timer,
646 			jiffies + pkc->tov_in_jiffies);
647 	pkc->last_kactive_blk_num = pkc->kactive_blk_num;
648 }
649 
650 /*
651  * Timer logic:
652  * 1) We refresh the timer only when we open a block.
653  *    By doing this we don't waste cycles refreshing the timer
654  *	  on packet-by-packet basis.
655  *
656  * With a 1MB block-size, on a 1Gbps line, it will take
657  * i) ~8 ms to fill a block + ii) memcpy etc.
658  * In this cut we are not accounting for the memcpy time.
659  *
660  * So, if the user sets the 'tmo' to 10ms then the timer
661  * will never fire while the block is still getting filled
662  * (which is what we want). However, the user could choose
663  * to close a block early and that's fine.
664  *
665  * But when the timer does fire, we check whether or not to refresh it.
666  * Since the tmo granularity is in msecs, it is not too expensive
667  * to refresh the timer, lets say every '8' msecs.
668  * Either the user can set the 'tmo' or we can derive it based on
669  * a) line-speed and b) block-size.
670  * prb_calc_retire_blk_tmo() calculates the tmo.
671  *
672  */
673 static void prb_retire_rx_blk_timer_expired(struct timer_list *t)
674 {
675 	struct packet_sock *po =
676 		from_timer(po, t, rx_ring.prb_bdqc.retire_blk_timer);
677 	struct tpacket_kbdq_core *pkc = GET_PBDQC_FROM_RB(&po->rx_ring);
678 	unsigned int frozen;
679 	struct tpacket_block_desc *pbd;
680 
681 	spin_lock(&po->sk.sk_receive_queue.lock);
682 
683 	frozen = prb_queue_frozen(pkc);
684 	pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
685 
686 	if (unlikely(pkc->delete_blk_timer))
687 		goto out;
688 
689 	/* We only need to plug the race when the block is partially filled.
690 	 * tpacket_rcv:
691 	 *		lock(); increment BLOCK_NUM_PKTS; unlock()
692 	 *		copy_bits() is in progress ...
693 	 *		timer fires on other cpu:
694 	 *		we can't retire the current block because copy_bits
695 	 *		is in progress.
696 	 *
697 	 */
698 	if (BLOCK_NUM_PKTS(pbd)) {
699 		while (atomic_read(&pkc->blk_fill_in_prog)) {
700 			/* Waiting for skb_copy_bits to finish... */
701 			cpu_relax();
702 		}
703 	}
704 
705 	if (pkc->last_kactive_blk_num == pkc->kactive_blk_num) {
706 		if (!frozen) {
707 			if (!BLOCK_NUM_PKTS(pbd)) {
708 				/* An empty block. Just refresh the timer. */
709 				goto refresh_timer;
710 			}
711 			prb_retire_current_block(pkc, po, TP_STATUS_BLK_TMO);
712 			if (!prb_dispatch_next_block(pkc, po))
713 				goto refresh_timer;
714 			else
715 				goto out;
716 		} else {
717 			/* Case 1. Queue was frozen because user-space was
718 			 *	   lagging behind.
719 			 */
720 			if (prb_curr_blk_in_use(pbd)) {
721 				/*
722 				 * Ok, user-space is still behind.
723 				 * So just refresh the timer.
724 				 */
725 				goto refresh_timer;
726 			} else {
727 			       /* Case 2. queue was frozen,user-space caught up,
728 				* now the link went idle && the timer fired.
729 				* We don't have a block to close.So we open this
730 				* block and restart the timer.
731 				* opening a block thaws the queue,restarts timer
732 				* Thawing/timer-refresh is a side effect.
733 				*/
734 				prb_open_block(pkc, pbd);
735 				goto out;
736 			}
737 		}
738 	}
739 
740 refresh_timer:
741 	_prb_refresh_rx_retire_blk_timer(pkc);
742 
743 out:
744 	spin_unlock(&po->sk.sk_receive_queue.lock);
745 }
746 
747 static void prb_flush_block(struct tpacket_kbdq_core *pkc1,
748 		struct tpacket_block_desc *pbd1, __u32 status)
749 {
750 	/* Flush everything minus the block header */
751 
752 #if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE == 1
753 	u8 *start, *end;
754 
755 	start = (u8 *)pbd1;
756 
757 	/* Skip the block header(we know header WILL fit in 4K) */
758 	start += PAGE_SIZE;
759 
760 	end = (u8 *)PAGE_ALIGN((unsigned long)pkc1->pkblk_end);
761 	for (; start < end; start += PAGE_SIZE)
762 		flush_dcache_page(pgv_to_page(start));
763 
764 	smp_wmb();
765 #endif
766 
767 	/* Now update the block status. */
768 
769 	BLOCK_STATUS(pbd1) = status;
770 
771 	/* Flush the block header */
772 
773 #if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE == 1
774 	start = (u8 *)pbd1;
775 	flush_dcache_page(pgv_to_page(start));
776 
777 	smp_wmb();
778 #endif
779 }
780 
781 /*
782  * Side effect:
783  *
784  * 1) flush the block
785  * 2) Increment active_blk_num
786  *
787  * Note:We DONT refresh the timer on purpose.
788  *	Because almost always the next block will be opened.
789  */
790 static void prb_close_block(struct tpacket_kbdq_core *pkc1,
791 		struct tpacket_block_desc *pbd1,
792 		struct packet_sock *po, unsigned int stat)
793 {
794 	__u32 status = TP_STATUS_USER | stat;
795 
796 	struct tpacket3_hdr *last_pkt;
797 	struct tpacket_hdr_v1 *h1 = &pbd1->hdr.bh1;
798 	struct sock *sk = &po->sk;
799 
800 	if (po->stats.stats3.tp_drops)
801 		status |= TP_STATUS_LOSING;
802 
803 	last_pkt = (struct tpacket3_hdr *)pkc1->prev;
804 	last_pkt->tp_next_offset = 0;
805 
806 	/* Get the ts of the last pkt */
807 	if (BLOCK_NUM_PKTS(pbd1)) {
808 		h1->ts_last_pkt.ts_sec = last_pkt->tp_sec;
809 		h1->ts_last_pkt.ts_nsec	= last_pkt->tp_nsec;
810 	} else {
811 		/* Ok, we tmo'd - so get the current time.
812 		 *
813 		 * It shouldn't really happen as we don't close empty
814 		 * blocks. See prb_retire_rx_blk_timer_expired().
815 		 */
816 		struct timespec ts;
817 		getnstimeofday(&ts);
818 		h1->ts_last_pkt.ts_sec = ts.tv_sec;
819 		h1->ts_last_pkt.ts_nsec	= ts.tv_nsec;
820 	}
821 
822 	smp_wmb();
823 
824 	/* Flush the block */
825 	prb_flush_block(pkc1, pbd1, status);
826 
827 	sk->sk_data_ready(sk);
828 
829 	pkc1->kactive_blk_num = GET_NEXT_PRB_BLK_NUM(pkc1);
830 }
831 
832 static void prb_thaw_queue(struct tpacket_kbdq_core *pkc)
833 {
834 	pkc->reset_pending_on_curr_blk = 0;
835 }
836 
837 /*
838  * Side effect of opening a block:
839  *
840  * 1) prb_queue is thawed.
841  * 2) retire_blk_timer is refreshed.
842  *
843  */
844 static void prb_open_block(struct tpacket_kbdq_core *pkc1,
845 	struct tpacket_block_desc *pbd1)
846 {
847 	struct timespec ts;
848 	struct tpacket_hdr_v1 *h1 = &pbd1->hdr.bh1;
849 
850 	smp_rmb();
851 
852 	/* We could have just memset this but we will lose the
853 	 * flexibility of making the priv area sticky
854 	 */
855 
856 	BLOCK_SNUM(pbd1) = pkc1->knxt_seq_num++;
857 	BLOCK_NUM_PKTS(pbd1) = 0;
858 	BLOCK_LEN(pbd1) = BLK_PLUS_PRIV(pkc1->blk_sizeof_priv);
859 
860 	getnstimeofday(&ts);
861 
862 	h1->ts_first_pkt.ts_sec = ts.tv_sec;
863 	h1->ts_first_pkt.ts_nsec = ts.tv_nsec;
864 
865 	pkc1->pkblk_start = (char *)pbd1;
866 	pkc1->nxt_offset = pkc1->pkblk_start + BLK_PLUS_PRIV(pkc1->blk_sizeof_priv);
867 
868 	BLOCK_O2FP(pbd1) = (__u32)BLK_PLUS_PRIV(pkc1->blk_sizeof_priv);
869 	BLOCK_O2PRIV(pbd1) = BLK_HDR_LEN;
870 
871 	pbd1->version = pkc1->version;
872 	pkc1->prev = pkc1->nxt_offset;
873 	pkc1->pkblk_end = pkc1->pkblk_start + pkc1->kblk_size;
874 
875 	prb_thaw_queue(pkc1);
876 	_prb_refresh_rx_retire_blk_timer(pkc1);
877 
878 	smp_wmb();
879 }
880 
881 /*
882  * Queue freeze logic:
883  * 1) Assume tp_block_nr = 8 blocks.
884  * 2) At time 't0', user opens Rx ring.
885  * 3) Some time past 't0', kernel starts filling blocks starting from 0 .. 7
886  * 4) user-space is either sleeping or processing block '0'.
887  * 5) tpacket_rcv is currently filling block '7', since there is no space left,
888  *    it will close block-7,loop around and try to fill block '0'.
889  *    call-flow:
890  *    __packet_lookup_frame_in_block
891  *      prb_retire_current_block()
892  *      prb_dispatch_next_block()
893  *        |->(BLOCK_STATUS == USER) evaluates to true
894  *    5.1) Since block-0 is currently in-use, we just freeze the queue.
895  * 6) Now there are two cases:
896  *    6.1) Link goes idle right after the queue is frozen.
897  *         But remember, the last open_block() refreshed the timer.
898  *         When this timer expires,it will refresh itself so that we can
899  *         re-open block-0 in near future.
900  *    6.2) Link is busy and keeps on receiving packets. This is a simple
901  *         case and __packet_lookup_frame_in_block will check if block-0
902  *         is free and can now be re-used.
903  */
904 static void prb_freeze_queue(struct tpacket_kbdq_core *pkc,
905 				  struct packet_sock *po)
906 {
907 	pkc->reset_pending_on_curr_blk = 1;
908 	po->stats.stats3.tp_freeze_q_cnt++;
909 }
910 
911 #define TOTAL_PKT_LEN_INCL_ALIGN(length) (ALIGN((length), V3_ALIGNMENT))
912 
913 /*
914  * If the next block is free then we will dispatch it
915  * and return a good offset.
916  * Else, we will freeze the queue.
917  * So, caller must check the return value.
918  */
919 static void *prb_dispatch_next_block(struct tpacket_kbdq_core *pkc,
920 		struct packet_sock *po)
921 {
922 	struct tpacket_block_desc *pbd;
923 
924 	smp_rmb();
925 
926 	/* 1. Get current block num */
927 	pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
928 
929 	/* 2. If this block is currently in_use then freeze the queue */
930 	if (TP_STATUS_USER & BLOCK_STATUS(pbd)) {
931 		prb_freeze_queue(pkc, po);
932 		return NULL;
933 	}
934 
935 	/*
936 	 * 3.
937 	 * open this block and return the offset where the first packet
938 	 * needs to get stored.
939 	 */
940 	prb_open_block(pkc, pbd);
941 	return (void *)pkc->nxt_offset;
942 }
943 
944 static void prb_retire_current_block(struct tpacket_kbdq_core *pkc,
945 		struct packet_sock *po, unsigned int status)
946 {
947 	struct tpacket_block_desc *pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
948 
949 	/* retire/close the current block */
950 	if (likely(TP_STATUS_KERNEL == BLOCK_STATUS(pbd))) {
951 		/*
952 		 * Plug the case where copy_bits() is in progress on
953 		 * cpu-0 and tpacket_rcv() got invoked on cpu-1, didn't
954 		 * have space to copy the pkt in the current block and
955 		 * called prb_retire_current_block()
956 		 *
957 		 * We don't need to worry about the TMO case because
958 		 * the timer-handler already handled this case.
959 		 */
960 		if (!(status & TP_STATUS_BLK_TMO)) {
961 			while (atomic_read(&pkc->blk_fill_in_prog)) {
962 				/* Waiting for skb_copy_bits to finish... */
963 				cpu_relax();
964 			}
965 		}
966 		prb_close_block(pkc, pbd, po, status);
967 		return;
968 	}
969 }
970 
971 static int prb_curr_blk_in_use(struct tpacket_block_desc *pbd)
972 {
973 	return TP_STATUS_USER & BLOCK_STATUS(pbd);
974 }
975 
976 static int prb_queue_frozen(struct tpacket_kbdq_core *pkc)
977 {
978 	return pkc->reset_pending_on_curr_blk;
979 }
980 
981 static void prb_clear_blk_fill_status(struct packet_ring_buffer *rb)
982 {
983 	struct tpacket_kbdq_core *pkc  = GET_PBDQC_FROM_RB(rb);
984 	atomic_dec(&pkc->blk_fill_in_prog);
985 }
986 
987 static void prb_fill_rxhash(struct tpacket_kbdq_core *pkc,
988 			struct tpacket3_hdr *ppd)
989 {
990 	ppd->hv1.tp_rxhash = skb_get_hash(pkc->skb);
991 }
992 
993 static void prb_clear_rxhash(struct tpacket_kbdq_core *pkc,
994 			struct tpacket3_hdr *ppd)
995 {
996 	ppd->hv1.tp_rxhash = 0;
997 }
998 
999 static void prb_fill_vlan_info(struct tpacket_kbdq_core *pkc,
1000 			struct tpacket3_hdr *ppd)
1001 {
1002 	if (skb_vlan_tag_present(pkc->skb)) {
1003 		ppd->hv1.tp_vlan_tci = skb_vlan_tag_get(pkc->skb);
1004 		ppd->hv1.tp_vlan_tpid = ntohs(pkc->skb->vlan_proto);
1005 		ppd->tp_status = TP_STATUS_VLAN_VALID | TP_STATUS_VLAN_TPID_VALID;
1006 	} else {
1007 		ppd->hv1.tp_vlan_tci = 0;
1008 		ppd->hv1.tp_vlan_tpid = 0;
1009 		ppd->tp_status = TP_STATUS_AVAILABLE;
1010 	}
1011 }
1012 
1013 static void prb_run_all_ft_ops(struct tpacket_kbdq_core *pkc,
1014 			struct tpacket3_hdr *ppd)
1015 {
1016 	ppd->hv1.tp_padding = 0;
1017 	prb_fill_vlan_info(pkc, ppd);
1018 
1019 	if (pkc->feature_req_word & TP_FT_REQ_FILL_RXHASH)
1020 		prb_fill_rxhash(pkc, ppd);
1021 	else
1022 		prb_clear_rxhash(pkc, ppd);
1023 }
1024 
1025 static void prb_fill_curr_block(char *curr,
1026 				struct tpacket_kbdq_core *pkc,
1027 				struct tpacket_block_desc *pbd,
1028 				unsigned int len)
1029 {
1030 	struct tpacket3_hdr *ppd;
1031 
1032 	ppd  = (struct tpacket3_hdr *)curr;
1033 	ppd->tp_next_offset = TOTAL_PKT_LEN_INCL_ALIGN(len);
1034 	pkc->prev = curr;
1035 	pkc->nxt_offset += TOTAL_PKT_LEN_INCL_ALIGN(len);
1036 	BLOCK_LEN(pbd) += TOTAL_PKT_LEN_INCL_ALIGN(len);
1037 	BLOCK_NUM_PKTS(pbd) += 1;
1038 	atomic_inc(&pkc->blk_fill_in_prog);
1039 	prb_run_all_ft_ops(pkc, ppd);
1040 }
1041 
1042 /* Assumes caller has the sk->rx_queue.lock */
1043 static void *__packet_lookup_frame_in_block(struct packet_sock *po,
1044 					    struct sk_buff *skb,
1045 						int status,
1046 					    unsigned int len
1047 					    )
1048 {
1049 	struct tpacket_kbdq_core *pkc;
1050 	struct tpacket_block_desc *pbd;
1051 	char *curr, *end;
1052 
1053 	pkc = GET_PBDQC_FROM_RB(&po->rx_ring);
1054 	pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
1055 
1056 	/* Queue is frozen when user space is lagging behind */
1057 	if (prb_queue_frozen(pkc)) {
1058 		/*
1059 		 * Check if that last block which caused the queue to freeze,
1060 		 * is still in_use by user-space.
1061 		 */
1062 		if (prb_curr_blk_in_use(pbd)) {
1063 			/* Can't record this packet */
1064 			return NULL;
1065 		} else {
1066 			/*
1067 			 * Ok, the block was released by user-space.
1068 			 * Now let's open that block.
1069 			 * opening a block also thaws the queue.
1070 			 * Thawing is a side effect.
1071 			 */
1072 			prb_open_block(pkc, pbd);
1073 		}
1074 	}
1075 
1076 	smp_mb();
1077 	curr = pkc->nxt_offset;
1078 	pkc->skb = skb;
1079 	end = (char *)pbd + pkc->kblk_size;
1080 
1081 	/* first try the current block */
1082 	if (curr+TOTAL_PKT_LEN_INCL_ALIGN(len) < end) {
1083 		prb_fill_curr_block(curr, pkc, pbd, len);
1084 		return (void *)curr;
1085 	}
1086 
1087 	/* Ok, close the current block */
1088 	prb_retire_current_block(pkc, po, 0);
1089 
1090 	/* Now, try to dispatch the next block */
1091 	curr = (char *)prb_dispatch_next_block(pkc, po);
1092 	if (curr) {
1093 		pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
1094 		prb_fill_curr_block(curr, pkc, pbd, len);
1095 		return (void *)curr;
1096 	}
1097 
1098 	/*
1099 	 * No free blocks are available.user_space hasn't caught up yet.
1100 	 * Queue was just frozen and now this packet will get dropped.
1101 	 */
1102 	return NULL;
1103 }
1104 
1105 static void *packet_current_rx_frame(struct packet_sock *po,
1106 					    struct sk_buff *skb,
1107 					    int status, unsigned int len)
1108 {
1109 	char *curr = NULL;
1110 	switch (po->tp_version) {
1111 	case TPACKET_V1:
1112 	case TPACKET_V2:
1113 		curr = packet_lookup_frame(po, &po->rx_ring,
1114 					po->rx_ring.head, status);
1115 		return curr;
1116 	case TPACKET_V3:
1117 		return __packet_lookup_frame_in_block(po, skb, status, len);
1118 	default:
1119 		WARN(1, "TPACKET version not supported\n");
1120 		BUG();
1121 		return NULL;
1122 	}
1123 }
1124 
1125 static void *prb_lookup_block(struct packet_sock *po,
1126 				     struct packet_ring_buffer *rb,
1127 				     unsigned int idx,
1128 				     int status)
1129 {
1130 	struct tpacket_kbdq_core *pkc  = GET_PBDQC_FROM_RB(rb);
1131 	struct tpacket_block_desc *pbd = GET_PBLOCK_DESC(pkc, idx);
1132 
1133 	if (status != BLOCK_STATUS(pbd))
1134 		return NULL;
1135 	return pbd;
1136 }
1137 
1138 static int prb_previous_blk_num(struct packet_ring_buffer *rb)
1139 {
1140 	unsigned int prev;
1141 	if (rb->prb_bdqc.kactive_blk_num)
1142 		prev = rb->prb_bdqc.kactive_blk_num-1;
1143 	else
1144 		prev = rb->prb_bdqc.knum_blocks-1;
1145 	return prev;
1146 }
1147 
1148 /* Assumes caller has held the rx_queue.lock */
1149 static void *__prb_previous_block(struct packet_sock *po,
1150 					 struct packet_ring_buffer *rb,
1151 					 int status)
1152 {
1153 	unsigned int previous = prb_previous_blk_num(rb);
1154 	return prb_lookup_block(po, rb, previous, status);
1155 }
1156 
1157 static void *packet_previous_rx_frame(struct packet_sock *po,
1158 					     struct packet_ring_buffer *rb,
1159 					     int status)
1160 {
1161 	if (po->tp_version <= TPACKET_V2)
1162 		return packet_previous_frame(po, rb, status);
1163 
1164 	return __prb_previous_block(po, rb, status);
1165 }
1166 
1167 static void packet_increment_rx_head(struct packet_sock *po,
1168 					    struct packet_ring_buffer *rb)
1169 {
1170 	switch (po->tp_version) {
1171 	case TPACKET_V1:
1172 	case TPACKET_V2:
1173 		return packet_increment_head(rb);
1174 	case TPACKET_V3:
1175 	default:
1176 		WARN(1, "TPACKET version not supported.\n");
1177 		BUG();
1178 		return;
1179 	}
1180 }
1181 
1182 static void *packet_previous_frame(struct packet_sock *po,
1183 		struct packet_ring_buffer *rb,
1184 		int status)
1185 {
1186 	unsigned int previous = rb->head ? rb->head - 1 : rb->frame_max;
1187 	return packet_lookup_frame(po, rb, previous, status);
1188 }
1189 
1190 static void packet_increment_head(struct packet_ring_buffer *buff)
1191 {
1192 	buff->head = buff->head != buff->frame_max ? buff->head+1 : 0;
1193 }
1194 
1195 static void packet_inc_pending(struct packet_ring_buffer *rb)
1196 {
1197 	this_cpu_inc(*rb->pending_refcnt);
1198 }
1199 
1200 static void packet_dec_pending(struct packet_ring_buffer *rb)
1201 {
1202 	this_cpu_dec(*rb->pending_refcnt);
1203 }
1204 
1205 static unsigned int packet_read_pending(const struct packet_ring_buffer *rb)
1206 {
1207 	unsigned int refcnt = 0;
1208 	int cpu;
1209 
1210 	/* We don't use pending refcount in rx_ring. */
1211 	if (rb->pending_refcnt == NULL)
1212 		return 0;
1213 
1214 	for_each_possible_cpu(cpu)
1215 		refcnt += *per_cpu_ptr(rb->pending_refcnt, cpu);
1216 
1217 	return refcnt;
1218 }
1219 
1220 static int packet_alloc_pending(struct packet_sock *po)
1221 {
1222 	po->rx_ring.pending_refcnt = NULL;
1223 
1224 	po->tx_ring.pending_refcnt = alloc_percpu(unsigned int);
1225 	if (unlikely(po->tx_ring.pending_refcnt == NULL))
1226 		return -ENOBUFS;
1227 
1228 	return 0;
1229 }
1230 
1231 static void packet_free_pending(struct packet_sock *po)
1232 {
1233 	free_percpu(po->tx_ring.pending_refcnt);
1234 }
1235 
1236 #define ROOM_POW_OFF	2
1237 #define ROOM_NONE	0x0
1238 #define ROOM_LOW	0x1
1239 #define ROOM_NORMAL	0x2
1240 
1241 static bool __tpacket_has_room(struct packet_sock *po, int pow_off)
1242 {
1243 	int idx, len;
1244 
1245 	len = po->rx_ring.frame_max + 1;
1246 	idx = po->rx_ring.head;
1247 	if (pow_off)
1248 		idx += len >> pow_off;
1249 	if (idx >= len)
1250 		idx -= len;
1251 	return packet_lookup_frame(po, &po->rx_ring, idx, TP_STATUS_KERNEL);
1252 }
1253 
1254 static bool __tpacket_v3_has_room(struct packet_sock *po, int pow_off)
1255 {
1256 	int idx, len;
1257 
1258 	len = po->rx_ring.prb_bdqc.knum_blocks;
1259 	idx = po->rx_ring.prb_bdqc.kactive_blk_num;
1260 	if (pow_off)
1261 		idx += len >> pow_off;
1262 	if (idx >= len)
1263 		idx -= len;
1264 	return prb_lookup_block(po, &po->rx_ring, idx, TP_STATUS_KERNEL);
1265 }
1266 
1267 static int __packet_rcv_has_room(struct packet_sock *po, struct sk_buff *skb)
1268 {
1269 	struct sock *sk = &po->sk;
1270 	int ret = ROOM_NONE;
1271 
1272 	if (po->prot_hook.func != tpacket_rcv) {
1273 		int avail = sk->sk_rcvbuf - atomic_read(&sk->sk_rmem_alloc)
1274 					  - (skb ? skb->truesize : 0);
1275 		if (avail > (sk->sk_rcvbuf >> ROOM_POW_OFF))
1276 			return ROOM_NORMAL;
1277 		else if (avail > 0)
1278 			return ROOM_LOW;
1279 		else
1280 			return ROOM_NONE;
1281 	}
1282 
1283 	if (po->tp_version == TPACKET_V3) {
1284 		if (__tpacket_v3_has_room(po, ROOM_POW_OFF))
1285 			ret = ROOM_NORMAL;
1286 		else if (__tpacket_v3_has_room(po, 0))
1287 			ret = ROOM_LOW;
1288 	} else {
1289 		if (__tpacket_has_room(po, ROOM_POW_OFF))
1290 			ret = ROOM_NORMAL;
1291 		else if (__tpacket_has_room(po, 0))
1292 			ret = ROOM_LOW;
1293 	}
1294 
1295 	return ret;
1296 }
1297 
1298 static int packet_rcv_has_room(struct packet_sock *po, struct sk_buff *skb)
1299 {
1300 	int ret;
1301 	bool has_room;
1302 
1303 	spin_lock_bh(&po->sk.sk_receive_queue.lock);
1304 	ret = __packet_rcv_has_room(po, skb);
1305 	has_room = ret == ROOM_NORMAL;
1306 	if (po->pressure == has_room)
1307 		po->pressure = !has_room;
1308 	spin_unlock_bh(&po->sk.sk_receive_queue.lock);
1309 
1310 	return ret;
1311 }
1312 
1313 static void packet_sock_destruct(struct sock *sk)
1314 {
1315 	skb_queue_purge(&sk->sk_error_queue);
1316 
1317 	WARN_ON(atomic_read(&sk->sk_rmem_alloc));
1318 	WARN_ON(refcount_read(&sk->sk_wmem_alloc));
1319 
1320 	if (!sock_flag(sk, SOCK_DEAD)) {
1321 		pr_err("Attempt to release alive packet socket: %p\n", sk);
1322 		return;
1323 	}
1324 
1325 	sk_refcnt_debug_dec(sk);
1326 }
1327 
1328 static bool fanout_flow_is_huge(struct packet_sock *po, struct sk_buff *skb)
1329 {
1330 	u32 rxhash;
1331 	int i, count = 0;
1332 
1333 	rxhash = skb_get_hash(skb);
1334 	for (i = 0; i < ROLLOVER_HLEN; i++)
1335 		if (po->rollover->history[i] == rxhash)
1336 			count++;
1337 
1338 	po->rollover->history[prandom_u32() % ROLLOVER_HLEN] = rxhash;
1339 	return count > (ROLLOVER_HLEN >> 1);
1340 }
1341 
1342 static unsigned int fanout_demux_hash(struct packet_fanout *f,
1343 				      struct sk_buff *skb,
1344 				      unsigned int num)
1345 {
1346 	return reciprocal_scale(__skb_get_hash_symmetric(skb), num);
1347 }
1348 
1349 static unsigned int fanout_demux_lb(struct packet_fanout *f,
1350 				    struct sk_buff *skb,
1351 				    unsigned int num)
1352 {
1353 	unsigned int val = atomic_inc_return(&f->rr_cur);
1354 
1355 	return val % num;
1356 }
1357 
1358 static unsigned int fanout_demux_cpu(struct packet_fanout *f,
1359 				     struct sk_buff *skb,
1360 				     unsigned int num)
1361 {
1362 	return smp_processor_id() % num;
1363 }
1364 
1365 static unsigned int fanout_demux_rnd(struct packet_fanout *f,
1366 				     struct sk_buff *skb,
1367 				     unsigned int num)
1368 {
1369 	return prandom_u32_max(num);
1370 }
1371 
1372 static unsigned int fanout_demux_rollover(struct packet_fanout *f,
1373 					  struct sk_buff *skb,
1374 					  unsigned int idx, bool try_self,
1375 					  unsigned int num)
1376 {
1377 	struct packet_sock *po, *po_next, *po_skip = NULL;
1378 	unsigned int i, j, room = ROOM_NONE;
1379 
1380 	po = pkt_sk(f->arr[idx]);
1381 
1382 	if (try_self) {
1383 		room = packet_rcv_has_room(po, skb);
1384 		if (room == ROOM_NORMAL ||
1385 		    (room == ROOM_LOW && !fanout_flow_is_huge(po, skb)))
1386 			return idx;
1387 		po_skip = po;
1388 	}
1389 
1390 	i = j = min_t(int, po->rollover->sock, num - 1);
1391 	do {
1392 		po_next = pkt_sk(f->arr[i]);
1393 		if (po_next != po_skip && !po_next->pressure &&
1394 		    packet_rcv_has_room(po_next, skb) == ROOM_NORMAL) {
1395 			if (i != j)
1396 				po->rollover->sock = i;
1397 			atomic_long_inc(&po->rollover->num);
1398 			if (room == ROOM_LOW)
1399 				atomic_long_inc(&po->rollover->num_huge);
1400 			return i;
1401 		}
1402 
1403 		if (++i == num)
1404 			i = 0;
1405 	} while (i != j);
1406 
1407 	atomic_long_inc(&po->rollover->num_failed);
1408 	return idx;
1409 }
1410 
1411 static unsigned int fanout_demux_qm(struct packet_fanout *f,
1412 				    struct sk_buff *skb,
1413 				    unsigned int num)
1414 {
1415 	return skb_get_queue_mapping(skb) % num;
1416 }
1417 
1418 static unsigned int fanout_demux_bpf(struct packet_fanout *f,
1419 				     struct sk_buff *skb,
1420 				     unsigned int num)
1421 {
1422 	struct bpf_prog *prog;
1423 	unsigned int ret = 0;
1424 
1425 	rcu_read_lock();
1426 	prog = rcu_dereference(f->bpf_prog);
1427 	if (prog)
1428 		ret = bpf_prog_run_clear_cb(prog, skb) % num;
1429 	rcu_read_unlock();
1430 
1431 	return ret;
1432 }
1433 
1434 static bool fanout_has_flag(struct packet_fanout *f, u16 flag)
1435 {
1436 	return f->flags & (flag >> 8);
1437 }
1438 
1439 static int packet_rcv_fanout(struct sk_buff *skb, struct net_device *dev,
1440 			     struct packet_type *pt, struct net_device *orig_dev)
1441 {
1442 	struct packet_fanout *f = pt->af_packet_priv;
1443 	unsigned int num = READ_ONCE(f->num_members);
1444 	struct net *net = read_pnet(&f->net);
1445 	struct packet_sock *po;
1446 	unsigned int idx;
1447 
1448 	if (!net_eq(dev_net(dev), net) || !num) {
1449 		kfree_skb(skb);
1450 		return 0;
1451 	}
1452 
1453 	if (fanout_has_flag(f, PACKET_FANOUT_FLAG_DEFRAG)) {
1454 		skb = ip_check_defrag(net, skb, IP_DEFRAG_AF_PACKET);
1455 		if (!skb)
1456 			return 0;
1457 	}
1458 	switch (f->type) {
1459 	case PACKET_FANOUT_HASH:
1460 	default:
1461 		idx = fanout_demux_hash(f, skb, num);
1462 		break;
1463 	case PACKET_FANOUT_LB:
1464 		idx = fanout_demux_lb(f, skb, num);
1465 		break;
1466 	case PACKET_FANOUT_CPU:
1467 		idx = fanout_demux_cpu(f, skb, num);
1468 		break;
1469 	case PACKET_FANOUT_RND:
1470 		idx = fanout_demux_rnd(f, skb, num);
1471 		break;
1472 	case PACKET_FANOUT_QM:
1473 		idx = fanout_demux_qm(f, skb, num);
1474 		break;
1475 	case PACKET_FANOUT_ROLLOVER:
1476 		idx = fanout_demux_rollover(f, skb, 0, false, num);
1477 		break;
1478 	case PACKET_FANOUT_CBPF:
1479 	case PACKET_FANOUT_EBPF:
1480 		idx = fanout_demux_bpf(f, skb, num);
1481 		break;
1482 	}
1483 
1484 	if (fanout_has_flag(f, PACKET_FANOUT_FLAG_ROLLOVER))
1485 		idx = fanout_demux_rollover(f, skb, idx, true, num);
1486 
1487 	po = pkt_sk(f->arr[idx]);
1488 	return po->prot_hook.func(skb, dev, &po->prot_hook, orig_dev);
1489 }
1490 
1491 DEFINE_MUTEX(fanout_mutex);
1492 EXPORT_SYMBOL_GPL(fanout_mutex);
1493 static LIST_HEAD(fanout_list);
1494 static u16 fanout_next_id;
1495 
1496 static void __fanout_link(struct sock *sk, struct packet_sock *po)
1497 {
1498 	struct packet_fanout *f = po->fanout;
1499 
1500 	spin_lock(&f->lock);
1501 	f->arr[f->num_members] = sk;
1502 	smp_wmb();
1503 	f->num_members++;
1504 	if (f->num_members == 1)
1505 		dev_add_pack(&f->prot_hook);
1506 	spin_unlock(&f->lock);
1507 }
1508 
1509 static void __fanout_unlink(struct sock *sk, struct packet_sock *po)
1510 {
1511 	struct packet_fanout *f = po->fanout;
1512 	int i;
1513 
1514 	spin_lock(&f->lock);
1515 	for (i = 0; i < f->num_members; i++) {
1516 		if (f->arr[i] == sk)
1517 			break;
1518 	}
1519 	BUG_ON(i >= f->num_members);
1520 	f->arr[i] = f->arr[f->num_members - 1];
1521 	f->num_members--;
1522 	if (f->num_members == 0)
1523 		__dev_remove_pack(&f->prot_hook);
1524 	spin_unlock(&f->lock);
1525 }
1526 
1527 static bool match_fanout_group(struct packet_type *ptype, struct sock *sk)
1528 {
1529 	if (sk->sk_family != PF_PACKET)
1530 		return false;
1531 
1532 	return ptype->af_packet_priv == pkt_sk(sk)->fanout;
1533 }
1534 
1535 static void fanout_init_data(struct packet_fanout *f)
1536 {
1537 	switch (f->type) {
1538 	case PACKET_FANOUT_LB:
1539 		atomic_set(&f->rr_cur, 0);
1540 		break;
1541 	case PACKET_FANOUT_CBPF:
1542 	case PACKET_FANOUT_EBPF:
1543 		RCU_INIT_POINTER(f->bpf_prog, NULL);
1544 		break;
1545 	}
1546 }
1547 
1548 static void __fanout_set_data_bpf(struct packet_fanout *f, struct bpf_prog *new)
1549 {
1550 	struct bpf_prog *old;
1551 
1552 	spin_lock(&f->lock);
1553 	old = rcu_dereference_protected(f->bpf_prog, lockdep_is_held(&f->lock));
1554 	rcu_assign_pointer(f->bpf_prog, new);
1555 	spin_unlock(&f->lock);
1556 
1557 	if (old) {
1558 		synchronize_net();
1559 		bpf_prog_destroy(old);
1560 	}
1561 }
1562 
1563 static int fanout_set_data_cbpf(struct packet_sock *po, char __user *data,
1564 				unsigned int len)
1565 {
1566 	struct bpf_prog *new;
1567 	struct sock_fprog fprog;
1568 	int ret;
1569 
1570 	if (sock_flag(&po->sk, SOCK_FILTER_LOCKED))
1571 		return -EPERM;
1572 	if (len != sizeof(fprog))
1573 		return -EINVAL;
1574 	if (copy_from_user(&fprog, data, len))
1575 		return -EFAULT;
1576 
1577 	ret = bpf_prog_create_from_user(&new, &fprog, NULL, false);
1578 	if (ret)
1579 		return ret;
1580 
1581 	__fanout_set_data_bpf(po->fanout, new);
1582 	return 0;
1583 }
1584 
1585 static int fanout_set_data_ebpf(struct packet_sock *po, char __user *data,
1586 				unsigned int len)
1587 {
1588 	struct bpf_prog *new;
1589 	u32 fd;
1590 
1591 	if (sock_flag(&po->sk, SOCK_FILTER_LOCKED))
1592 		return -EPERM;
1593 	if (len != sizeof(fd))
1594 		return -EINVAL;
1595 	if (copy_from_user(&fd, data, len))
1596 		return -EFAULT;
1597 
1598 	new = bpf_prog_get_type(fd, BPF_PROG_TYPE_SOCKET_FILTER);
1599 	if (IS_ERR(new))
1600 		return PTR_ERR(new);
1601 
1602 	__fanout_set_data_bpf(po->fanout, new);
1603 	return 0;
1604 }
1605 
1606 static int fanout_set_data(struct packet_sock *po, char __user *data,
1607 			   unsigned int len)
1608 {
1609 	switch (po->fanout->type) {
1610 	case PACKET_FANOUT_CBPF:
1611 		return fanout_set_data_cbpf(po, data, len);
1612 	case PACKET_FANOUT_EBPF:
1613 		return fanout_set_data_ebpf(po, data, len);
1614 	default:
1615 		return -EINVAL;
1616 	};
1617 }
1618 
1619 static void fanout_release_data(struct packet_fanout *f)
1620 {
1621 	switch (f->type) {
1622 	case PACKET_FANOUT_CBPF:
1623 	case PACKET_FANOUT_EBPF:
1624 		__fanout_set_data_bpf(f, NULL);
1625 	};
1626 }
1627 
1628 static bool __fanout_id_is_free(struct sock *sk, u16 candidate_id)
1629 {
1630 	struct packet_fanout *f;
1631 
1632 	list_for_each_entry(f, &fanout_list, list) {
1633 		if (f->id == candidate_id &&
1634 		    read_pnet(&f->net) == sock_net(sk)) {
1635 			return false;
1636 		}
1637 	}
1638 	return true;
1639 }
1640 
1641 static bool fanout_find_new_id(struct sock *sk, u16 *new_id)
1642 {
1643 	u16 id = fanout_next_id;
1644 
1645 	do {
1646 		if (__fanout_id_is_free(sk, id)) {
1647 			*new_id = id;
1648 			fanout_next_id = id + 1;
1649 			return true;
1650 		}
1651 
1652 		id++;
1653 	} while (id != fanout_next_id);
1654 
1655 	return false;
1656 }
1657 
1658 static int fanout_add(struct sock *sk, u16 id, u16 type_flags)
1659 {
1660 	struct packet_rollover *rollover = NULL;
1661 	struct packet_sock *po = pkt_sk(sk);
1662 	struct packet_fanout *f, *match;
1663 	u8 type = type_flags & 0xff;
1664 	u8 flags = type_flags >> 8;
1665 	int err;
1666 
1667 	switch (type) {
1668 	case PACKET_FANOUT_ROLLOVER:
1669 		if (type_flags & PACKET_FANOUT_FLAG_ROLLOVER)
1670 			return -EINVAL;
1671 	case PACKET_FANOUT_HASH:
1672 	case PACKET_FANOUT_LB:
1673 	case PACKET_FANOUT_CPU:
1674 	case PACKET_FANOUT_RND:
1675 	case PACKET_FANOUT_QM:
1676 	case PACKET_FANOUT_CBPF:
1677 	case PACKET_FANOUT_EBPF:
1678 		break;
1679 	default:
1680 		return -EINVAL;
1681 	}
1682 
1683 	mutex_lock(&fanout_mutex);
1684 
1685 	err = -EALREADY;
1686 	if (po->fanout)
1687 		goto out;
1688 
1689 	if (type == PACKET_FANOUT_ROLLOVER ||
1690 	    (type_flags & PACKET_FANOUT_FLAG_ROLLOVER)) {
1691 		err = -ENOMEM;
1692 		rollover = kzalloc(sizeof(*rollover), GFP_KERNEL);
1693 		if (!rollover)
1694 			goto out;
1695 		atomic_long_set(&rollover->num, 0);
1696 		atomic_long_set(&rollover->num_huge, 0);
1697 		atomic_long_set(&rollover->num_failed, 0);
1698 	}
1699 
1700 	if (type_flags & PACKET_FANOUT_FLAG_UNIQUEID) {
1701 		if (id != 0) {
1702 			err = -EINVAL;
1703 			goto out;
1704 		}
1705 		if (!fanout_find_new_id(sk, &id)) {
1706 			err = -ENOMEM;
1707 			goto out;
1708 		}
1709 		/* ephemeral flag for the first socket in the group: drop it */
1710 		flags &= ~(PACKET_FANOUT_FLAG_UNIQUEID >> 8);
1711 	}
1712 
1713 	match = NULL;
1714 	list_for_each_entry(f, &fanout_list, list) {
1715 		if (f->id == id &&
1716 		    read_pnet(&f->net) == sock_net(sk)) {
1717 			match = f;
1718 			break;
1719 		}
1720 	}
1721 	err = -EINVAL;
1722 	if (match && match->flags != flags)
1723 		goto out;
1724 	if (!match) {
1725 		err = -ENOMEM;
1726 		match = kzalloc(sizeof(*match), GFP_KERNEL);
1727 		if (!match)
1728 			goto out;
1729 		write_pnet(&match->net, sock_net(sk));
1730 		match->id = id;
1731 		match->type = type;
1732 		match->flags = flags;
1733 		INIT_LIST_HEAD(&match->list);
1734 		spin_lock_init(&match->lock);
1735 		refcount_set(&match->sk_ref, 0);
1736 		fanout_init_data(match);
1737 		match->prot_hook.type = po->prot_hook.type;
1738 		match->prot_hook.dev = po->prot_hook.dev;
1739 		match->prot_hook.func = packet_rcv_fanout;
1740 		match->prot_hook.af_packet_priv = match;
1741 		match->prot_hook.id_match = match_fanout_group;
1742 		list_add(&match->list, &fanout_list);
1743 	}
1744 	err = -EINVAL;
1745 
1746 	spin_lock(&po->bind_lock);
1747 	if (po->running &&
1748 	    match->type == type &&
1749 	    match->prot_hook.type == po->prot_hook.type &&
1750 	    match->prot_hook.dev == po->prot_hook.dev) {
1751 		err = -ENOSPC;
1752 		if (refcount_read(&match->sk_ref) < PACKET_FANOUT_MAX) {
1753 			__dev_remove_pack(&po->prot_hook);
1754 			po->fanout = match;
1755 			po->rollover = rollover;
1756 			rollover = NULL;
1757 			refcount_set(&match->sk_ref, refcount_read(&match->sk_ref) + 1);
1758 			__fanout_link(sk, po);
1759 			err = 0;
1760 		}
1761 	}
1762 	spin_unlock(&po->bind_lock);
1763 
1764 	if (err && !refcount_read(&match->sk_ref)) {
1765 		list_del(&match->list);
1766 		kfree(match);
1767 	}
1768 
1769 out:
1770 	kfree(rollover);
1771 	mutex_unlock(&fanout_mutex);
1772 	return err;
1773 }
1774 
1775 /* If pkt_sk(sk)->fanout->sk_ref is zero, this function removes
1776  * pkt_sk(sk)->fanout from fanout_list and returns pkt_sk(sk)->fanout.
1777  * It is the responsibility of the caller to call fanout_release_data() and
1778  * free the returned packet_fanout (after synchronize_net())
1779  */
1780 static struct packet_fanout *fanout_release(struct sock *sk)
1781 {
1782 	struct packet_sock *po = pkt_sk(sk);
1783 	struct packet_fanout *f;
1784 
1785 	mutex_lock(&fanout_mutex);
1786 	f = po->fanout;
1787 	if (f) {
1788 		po->fanout = NULL;
1789 
1790 		if (refcount_dec_and_test(&f->sk_ref))
1791 			list_del(&f->list);
1792 		else
1793 			f = NULL;
1794 	}
1795 	mutex_unlock(&fanout_mutex);
1796 
1797 	return f;
1798 }
1799 
1800 static bool packet_extra_vlan_len_allowed(const struct net_device *dev,
1801 					  struct sk_buff *skb)
1802 {
1803 	/* Earlier code assumed this would be a VLAN pkt, double-check
1804 	 * this now that we have the actual packet in hand. We can only
1805 	 * do this check on Ethernet devices.
1806 	 */
1807 	if (unlikely(dev->type != ARPHRD_ETHER))
1808 		return false;
1809 
1810 	skb_reset_mac_header(skb);
1811 	return likely(eth_hdr(skb)->h_proto == htons(ETH_P_8021Q));
1812 }
1813 
1814 static const struct proto_ops packet_ops;
1815 
1816 static const struct proto_ops packet_ops_spkt;
1817 
1818 static int packet_rcv_spkt(struct sk_buff *skb, struct net_device *dev,
1819 			   struct packet_type *pt, struct net_device *orig_dev)
1820 {
1821 	struct sock *sk;
1822 	struct sockaddr_pkt *spkt;
1823 
1824 	/*
1825 	 *	When we registered the protocol we saved the socket in the data
1826 	 *	field for just this event.
1827 	 */
1828 
1829 	sk = pt->af_packet_priv;
1830 
1831 	/*
1832 	 *	Yank back the headers [hope the device set this
1833 	 *	right or kerboom...]
1834 	 *
1835 	 *	Incoming packets have ll header pulled,
1836 	 *	push it back.
1837 	 *
1838 	 *	For outgoing ones skb->data == skb_mac_header(skb)
1839 	 *	so that this procedure is noop.
1840 	 */
1841 
1842 	if (skb->pkt_type == PACKET_LOOPBACK)
1843 		goto out;
1844 
1845 	if (!net_eq(dev_net(dev), sock_net(sk)))
1846 		goto out;
1847 
1848 	skb = skb_share_check(skb, GFP_ATOMIC);
1849 	if (skb == NULL)
1850 		goto oom;
1851 
1852 	/* drop any routing info */
1853 	skb_dst_drop(skb);
1854 
1855 	/* drop conntrack reference */
1856 	nf_reset(skb);
1857 
1858 	spkt = &PACKET_SKB_CB(skb)->sa.pkt;
1859 
1860 	skb_push(skb, skb->data - skb_mac_header(skb));
1861 
1862 	/*
1863 	 *	The SOCK_PACKET socket receives _all_ frames.
1864 	 */
1865 
1866 	spkt->spkt_family = dev->type;
1867 	strlcpy(spkt->spkt_device, dev->name, sizeof(spkt->spkt_device));
1868 	spkt->spkt_protocol = skb->protocol;
1869 
1870 	/*
1871 	 *	Charge the memory to the socket. This is done specifically
1872 	 *	to prevent sockets using all the memory up.
1873 	 */
1874 
1875 	if (sock_queue_rcv_skb(sk, skb) == 0)
1876 		return 0;
1877 
1878 out:
1879 	kfree_skb(skb);
1880 oom:
1881 	return 0;
1882 }
1883 
1884 
1885 /*
1886  *	Output a raw packet to a device layer. This bypasses all the other
1887  *	protocol layers and you must therefore supply it with a complete frame
1888  */
1889 
1890 static int packet_sendmsg_spkt(struct socket *sock, struct msghdr *msg,
1891 			       size_t len)
1892 {
1893 	struct sock *sk = sock->sk;
1894 	DECLARE_SOCKADDR(struct sockaddr_pkt *, saddr, msg->msg_name);
1895 	struct sk_buff *skb = NULL;
1896 	struct net_device *dev;
1897 	struct sockcm_cookie sockc;
1898 	__be16 proto = 0;
1899 	int err;
1900 	int extra_len = 0;
1901 
1902 	/*
1903 	 *	Get and verify the address.
1904 	 */
1905 
1906 	if (saddr) {
1907 		if (msg->msg_namelen < sizeof(struct sockaddr))
1908 			return -EINVAL;
1909 		if (msg->msg_namelen == sizeof(struct sockaddr_pkt))
1910 			proto = saddr->spkt_protocol;
1911 	} else
1912 		return -ENOTCONN;	/* SOCK_PACKET must be sent giving an address */
1913 
1914 	/*
1915 	 *	Find the device first to size check it
1916 	 */
1917 
1918 	saddr->spkt_device[sizeof(saddr->spkt_device) - 1] = 0;
1919 retry:
1920 	rcu_read_lock();
1921 	dev = dev_get_by_name_rcu(sock_net(sk), saddr->spkt_device);
1922 	err = -ENODEV;
1923 	if (dev == NULL)
1924 		goto out_unlock;
1925 
1926 	err = -ENETDOWN;
1927 	if (!(dev->flags & IFF_UP))
1928 		goto out_unlock;
1929 
1930 	/*
1931 	 * You may not queue a frame bigger than the mtu. This is the lowest level
1932 	 * raw protocol and you must do your own fragmentation at this level.
1933 	 */
1934 
1935 	if (unlikely(sock_flag(sk, SOCK_NOFCS))) {
1936 		if (!netif_supports_nofcs(dev)) {
1937 			err = -EPROTONOSUPPORT;
1938 			goto out_unlock;
1939 		}
1940 		extra_len = 4; /* We're doing our own CRC */
1941 	}
1942 
1943 	err = -EMSGSIZE;
1944 	if (len > dev->mtu + dev->hard_header_len + VLAN_HLEN + extra_len)
1945 		goto out_unlock;
1946 
1947 	if (!skb) {
1948 		size_t reserved = LL_RESERVED_SPACE(dev);
1949 		int tlen = dev->needed_tailroom;
1950 		unsigned int hhlen = dev->header_ops ? dev->hard_header_len : 0;
1951 
1952 		rcu_read_unlock();
1953 		skb = sock_wmalloc(sk, len + reserved + tlen, 0, GFP_KERNEL);
1954 		if (skb == NULL)
1955 			return -ENOBUFS;
1956 		/* FIXME: Save some space for broken drivers that write a hard
1957 		 * header at transmission time by themselves. PPP is the notable
1958 		 * one here. This should really be fixed at the driver level.
1959 		 */
1960 		skb_reserve(skb, reserved);
1961 		skb_reset_network_header(skb);
1962 
1963 		/* Try to align data part correctly */
1964 		if (hhlen) {
1965 			skb->data -= hhlen;
1966 			skb->tail -= hhlen;
1967 			if (len < hhlen)
1968 				skb_reset_network_header(skb);
1969 		}
1970 		err = memcpy_from_msg(skb_put(skb, len), msg, len);
1971 		if (err)
1972 			goto out_free;
1973 		goto retry;
1974 	}
1975 
1976 	if (!dev_validate_header(dev, skb->data, len)) {
1977 		err = -EINVAL;
1978 		goto out_unlock;
1979 	}
1980 	if (len > (dev->mtu + dev->hard_header_len + extra_len) &&
1981 	    !packet_extra_vlan_len_allowed(dev, skb)) {
1982 		err = -EMSGSIZE;
1983 		goto out_unlock;
1984 	}
1985 
1986 	sockc.tsflags = sk->sk_tsflags;
1987 	if (msg->msg_controllen) {
1988 		err = sock_cmsg_send(sk, msg, &sockc);
1989 		if (unlikely(err))
1990 			goto out_unlock;
1991 	}
1992 
1993 	skb->protocol = proto;
1994 	skb->dev = dev;
1995 	skb->priority = sk->sk_priority;
1996 	skb->mark = sk->sk_mark;
1997 
1998 	sock_tx_timestamp(sk, sockc.tsflags, &skb_shinfo(skb)->tx_flags);
1999 
2000 	if (unlikely(extra_len == 4))
2001 		skb->no_fcs = 1;
2002 
2003 	skb_probe_transport_header(skb, 0);
2004 
2005 	dev_queue_xmit(skb);
2006 	rcu_read_unlock();
2007 	return len;
2008 
2009 out_unlock:
2010 	rcu_read_unlock();
2011 out_free:
2012 	kfree_skb(skb);
2013 	return err;
2014 }
2015 
2016 static unsigned int run_filter(struct sk_buff *skb,
2017 			       const struct sock *sk,
2018 			       unsigned int res)
2019 {
2020 	struct sk_filter *filter;
2021 
2022 	rcu_read_lock();
2023 	filter = rcu_dereference(sk->sk_filter);
2024 	if (filter != NULL)
2025 		res = bpf_prog_run_clear_cb(filter->prog, skb);
2026 	rcu_read_unlock();
2027 
2028 	return res;
2029 }
2030 
2031 static int packet_rcv_vnet(struct msghdr *msg, const struct sk_buff *skb,
2032 			   size_t *len)
2033 {
2034 	struct virtio_net_hdr vnet_hdr;
2035 
2036 	if (*len < sizeof(vnet_hdr))
2037 		return -EINVAL;
2038 	*len -= sizeof(vnet_hdr);
2039 
2040 	if (virtio_net_hdr_from_skb(skb, &vnet_hdr, vio_le(), true))
2041 		return -EINVAL;
2042 
2043 	return memcpy_to_msg(msg, (void *)&vnet_hdr, sizeof(vnet_hdr));
2044 }
2045 
2046 /*
2047  * This function makes lazy skb cloning in hope that most of packets
2048  * are discarded by BPF.
2049  *
2050  * Note tricky part: we DO mangle shared skb! skb->data, skb->len
2051  * and skb->cb are mangled. It works because (and until) packets
2052  * falling here are owned by current CPU. Output packets are cloned
2053  * by dev_queue_xmit_nit(), input packets are processed by net_bh
2054  * sequencially, so that if we return skb to original state on exit,
2055  * we will not harm anyone.
2056  */
2057 
2058 static int packet_rcv(struct sk_buff *skb, struct net_device *dev,
2059 		      struct packet_type *pt, struct net_device *orig_dev)
2060 {
2061 	struct sock *sk;
2062 	struct sockaddr_ll *sll;
2063 	struct packet_sock *po;
2064 	u8 *skb_head = skb->data;
2065 	int skb_len = skb->len;
2066 	unsigned int snaplen, res;
2067 	bool is_drop_n_account = false;
2068 
2069 	if (skb->pkt_type == PACKET_LOOPBACK)
2070 		goto drop;
2071 
2072 	sk = pt->af_packet_priv;
2073 	po = pkt_sk(sk);
2074 
2075 	if (!net_eq(dev_net(dev), sock_net(sk)))
2076 		goto drop;
2077 
2078 	skb->dev = dev;
2079 
2080 	if (dev->header_ops) {
2081 		/* The device has an explicit notion of ll header,
2082 		 * exported to higher levels.
2083 		 *
2084 		 * Otherwise, the device hides details of its frame
2085 		 * structure, so that corresponding packet head is
2086 		 * never delivered to user.
2087 		 */
2088 		if (sk->sk_type != SOCK_DGRAM)
2089 			skb_push(skb, skb->data - skb_mac_header(skb));
2090 		else if (skb->pkt_type == PACKET_OUTGOING) {
2091 			/* Special case: outgoing packets have ll header at head */
2092 			skb_pull(skb, skb_network_offset(skb));
2093 		}
2094 	}
2095 
2096 	snaplen = skb->len;
2097 
2098 	res = run_filter(skb, sk, snaplen);
2099 	if (!res)
2100 		goto drop_n_restore;
2101 	if (snaplen > res)
2102 		snaplen = res;
2103 
2104 	if (atomic_read(&sk->sk_rmem_alloc) >= sk->sk_rcvbuf)
2105 		goto drop_n_acct;
2106 
2107 	if (skb_shared(skb)) {
2108 		struct sk_buff *nskb = skb_clone(skb, GFP_ATOMIC);
2109 		if (nskb == NULL)
2110 			goto drop_n_acct;
2111 
2112 		if (skb_head != skb->data) {
2113 			skb->data = skb_head;
2114 			skb->len = skb_len;
2115 		}
2116 		consume_skb(skb);
2117 		skb = nskb;
2118 	}
2119 
2120 	sock_skb_cb_check_size(sizeof(*PACKET_SKB_CB(skb)) + MAX_ADDR_LEN - 8);
2121 
2122 	sll = &PACKET_SKB_CB(skb)->sa.ll;
2123 	sll->sll_hatype = dev->type;
2124 	sll->sll_pkttype = skb->pkt_type;
2125 	if (unlikely(po->origdev))
2126 		sll->sll_ifindex = orig_dev->ifindex;
2127 	else
2128 		sll->sll_ifindex = dev->ifindex;
2129 
2130 	sll->sll_halen = dev_parse_header(skb, sll->sll_addr);
2131 
2132 	/* sll->sll_family and sll->sll_protocol are set in packet_recvmsg().
2133 	 * Use their space for storing the original skb length.
2134 	 */
2135 	PACKET_SKB_CB(skb)->sa.origlen = skb->len;
2136 
2137 	if (pskb_trim(skb, snaplen))
2138 		goto drop_n_acct;
2139 
2140 	skb_set_owner_r(skb, sk);
2141 	skb->dev = NULL;
2142 	skb_dst_drop(skb);
2143 
2144 	/* drop conntrack reference */
2145 	nf_reset(skb);
2146 
2147 	spin_lock(&sk->sk_receive_queue.lock);
2148 	po->stats.stats1.tp_packets++;
2149 	sock_skb_set_dropcount(sk, skb);
2150 	__skb_queue_tail(&sk->sk_receive_queue, skb);
2151 	spin_unlock(&sk->sk_receive_queue.lock);
2152 	sk->sk_data_ready(sk);
2153 	return 0;
2154 
2155 drop_n_acct:
2156 	is_drop_n_account = true;
2157 	spin_lock(&sk->sk_receive_queue.lock);
2158 	po->stats.stats1.tp_drops++;
2159 	atomic_inc(&sk->sk_drops);
2160 	spin_unlock(&sk->sk_receive_queue.lock);
2161 
2162 drop_n_restore:
2163 	if (skb_head != skb->data && skb_shared(skb)) {
2164 		skb->data = skb_head;
2165 		skb->len = skb_len;
2166 	}
2167 drop:
2168 	if (!is_drop_n_account)
2169 		consume_skb(skb);
2170 	else
2171 		kfree_skb(skb);
2172 	return 0;
2173 }
2174 
2175 static int tpacket_rcv(struct sk_buff *skb, struct net_device *dev,
2176 		       struct packet_type *pt, struct net_device *orig_dev)
2177 {
2178 	struct sock *sk;
2179 	struct packet_sock *po;
2180 	struct sockaddr_ll *sll;
2181 	union tpacket_uhdr h;
2182 	u8 *skb_head = skb->data;
2183 	int skb_len = skb->len;
2184 	unsigned int snaplen, res;
2185 	unsigned long status = TP_STATUS_USER;
2186 	unsigned short macoff, netoff, hdrlen;
2187 	struct sk_buff *copy_skb = NULL;
2188 	struct timespec ts;
2189 	__u32 ts_status;
2190 	bool is_drop_n_account = false;
2191 	bool do_vnet = false;
2192 
2193 	/* struct tpacket{2,3}_hdr is aligned to a multiple of TPACKET_ALIGNMENT.
2194 	 * We may add members to them until current aligned size without forcing
2195 	 * userspace to call getsockopt(..., PACKET_HDRLEN, ...).
2196 	 */
2197 	BUILD_BUG_ON(TPACKET_ALIGN(sizeof(*h.h2)) != 32);
2198 	BUILD_BUG_ON(TPACKET_ALIGN(sizeof(*h.h3)) != 48);
2199 
2200 	if (skb->pkt_type == PACKET_LOOPBACK)
2201 		goto drop;
2202 
2203 	sk = pt->af_packet_priv;
2204 	po = pkt_sk(sk);
2205 
2206 	if (!net_eq(dev_net(dev), sock_net(sk)))
2207 		goto drop;
2208 
2209 	if (dev->header_ops) {
2210 		if (sk->sk_type != SOCK_DGRAM)
2211 			skb_push(skb, skb->data - skb_mac_header(skb));
2212 		else if (skb->pkt_type == PACKET_OUTGOING) {
2213 			/* Special case: outgoing packets have ll header at head */
2214 			skb_pull(skb, skb_network_offset(skb));
2215 		}
2216 	}
2217 
2218 	snaplen = skb->len;
2219 
2220 	res = run_filter(skb, sk, snaplen);
2221 	if (!res)
2222 		goto drop_n_restore;
2223 
2224 	if (skb->ip_summed == CHECKSUM_PARTIAL)
2225 		status |= TP_STATUS_CSUMNOTREADY;
2226 	else if (skb->pkt_type != PACKET_OUTGOING &&
2227 		 (skb->ip_summed == CHECKSUM_COMPLETE ||
2228 		  skb_csum_unnecessary(skb)))
2229 		status |= TP_STATUS_CSUM_VALID;
2230 
2231 	if (snaplen > res)
2232 		snaplen = res;
2233 
2234 	if (sk->sk_type == SOCK_DGRAM) {
2235 		macoff = netoff = TPACKET_ALIGN(po->tp_hdrlen) + 16 +
2236 				  po->tp_reserve;
2237 	} else {
2238 		unsigned int maclen = skb_network_offset(skb);
2239 		netoff = TPACKET_ALIGN(po->tp_hdrlen +
2240 				       (maclen < 16 ? 16 : maclen)) +
2241 				       po->tp_reserve;
2242 		if (po->has_vnet_hdr) {
2243 			netoff += sizeof(struct virtio_net_hdr);
2244 			do_vnet = true;
2245 		}
2246 		macoff = netoff - maclen;
2247 	}
2248 	if (po->tp_version <= TPACKET_V2) {
2249 		if (macoff + snaplen > po->rx_ring.frame_size) {
2250 			if (po->copy_thresh &&
2251 			    atomic_read(&sk->sk_rmem_alloc) < sk->sk_rcvbuf) {
2252 				if (skb_shared(skb)) {
2253 					copy_skb = skb_clone(skb, GFP_ATOMIC);
2254 				} else {
2255 					copy_skb = skb_get(skb);
2256 					skb_head = skb->data;
2257 				}
2258 				if (copy_skb)
2259 					skb_set_owner_r(copy_skb, sk);
2260 			}
2261 			snaplen = po->rx_ring.frame_size - macoff;
2262 			if ((int)snaplen < 0) {
2263 				snaplen = 0;
2264 				do_vnet = false;
2265 			}
2266 		}
2267 	} else if (unlikely(macoff + snaplen >
2268 			    GET_PBDQC_FROM_RB(&po->rx_ring)->max_frame_len)) {
2269 		u32 nval;
2270 
2271 		nval = GET_PBDQC_FROM_RB(&po->rx_ring)->max_frame_len - macoff;
2272 		pr_err_once("tpacket_rcv: packet too big, clamped from %u to %u. macoff=%u\n",
2273 			    snaplen, nval, macoff);
2274 		snaplen = nval;
2275 		if (unlikely((int)snaplen < 0)) {
2276 			snaplen = 0;
2277 			macoff = GET_PBDQC_FROM_RB(&po->rx_ring)->max_frame_len;
2278 			do_vnet = false;
2279 		}
2280 	}
2281 	spin_lock(&sk->sk_receive_queue.lock);
2282 	h.raw = packet_current_rx_frame(po, skb,
2283 					TP_STATUS_KERNEL, (macoff+snaplen));
2284 	if (!h.raw)
2285 		goto drop_n_account;
2286 	if (po->tp_version <= TPACKET_V2) {
2287 		packet_increment_rx_head(po, &po->rx_ring);
2288 	/*
2289 	 * LOSING will be reported till you read the stats,
2290 	 * because it's COR - Clear On Read.
2291 	 * Anyways, moving it for V1/V2 only as V3 doesn't need this
2292 	 * at packet level.
2293 	 */
2294 		if (po->stats.stats1.tp_drops)
2295 			status |= TP_STATUS_LOSING;
2296 	}
2297 	po->stats.stats1.tp_packets++;
2298 	if (copy_skb) {
2299 		status |= TP_STATUS_COPY;
2300 		__skb_queue_tail(&sk->sk_receive_queue, copy_skb);
2301 	}
2302 	spin_unlock(&sk->sk_receive_queue.lock);
2303 
2304 	if (do_vnet) {
2305 		if (virtio_net_hdr_from_skb(skb, h.raw + macoff -
2306 					    sizeof(struct virtio_net_hdr),
2307 					    vio_le(), true)) {
2308 			spin_lock(&sk->sk_receive_queue.lock);
2309 			goto drop_n_account;
2310 		}
2311 	}
2312 
2313 	skb_copy_bits(skb, 0, h.raw + macoff, snaplen);
2314 
2315 	if (!(ts_status = tpacket_get_timestamp(skb, &ts, po->tp_tstamp)))
2316 		getnstimeofday(&ts);
2317 
2318 	status |= ts_status;
2319 
2320 	switch (po->tp_version) {
2321 	case TPACKET_V1:
2322 		h.h1->tp_len = skb->len;
2323 		h.h1->tp_snaplen = snaplen;
2324 		h.h1->tp_mac = macoff;
2325 		h.h1->tp_net = netoff;
2326 		h.h1->tp_sec = ts.tv_sec;
2327 		h.h1->tp_usec = ts.tv_nsec / NSEC_PER_USEC;
2328 		hdrlen = sizeof(*h.h1);
2329 		break;
2330 	case TPACKET_V2:
2331 		h.h2->tp_len = skb->len;
2332 		h.h2->tp_snaplen = snaplen;
2333 		h.h2->tp_mac = macoff;
2334 		h.h2->tp_net = netoff;
2335 		h.h2->tp_sec = ts.tv_sec;
2336 		h.h2->tp_nsec = ts.tv_nsec;
2337 		if (skb_vlan_tag_present(skb)) {
2338 			h.h2->tp_vlan_tci = skb_vlan_tag_get(skb);
2339 			h.h2->tp_vlan_tpid = ntohs(skb->vlan_proto);
2340 			status |= TP_STATUS_VLAN_VALID | TP_STATUS_VLAN_TPID_VALID;
2341 		} else {
2342 			h.h2->tp_vlan_tci = 0;
2343 			h.h2->tp_vlan_tpid = 0;
2344 		}
2345 		memset(h.h2->tp_padding, 0, sizeof(h.h2->tp_padding));
2346 		hdrlen = sizeof(*h.h2);
2347 		break;
2348 	case TPACKET_V3:
2349 		/* tp_nxt_offset,vlan are already populated above.
2350 		 * So DONT clear those fields here
2351 		 */
2352 		h.h3->tp_status |= status;
2353 		h.h3->tp_len = skb->len;
2354 		h.h3->tp_snaplen = snaplen;
2355 		h.h3->tp_mac = macoff;
2356 		h.h3->tp_net = netoff;
2357 		h.h3->tp_sec  = ts.tv_sec;
2358 		h.h3->tp_nsec = ts.tv_nsec;
2359 		memset(h.h3->tp_padding, 0, sizeof(h.h3->tp_padding));
2360 		hdrlen = sizeof(*h.h3);
2361 		break;
2362 	default:
2363 		BUG();
2364 	}
2365 
2366 	sll = h.raw + TPACKET_ALIGN(hdrlen);
2367 	sll->sll_halen = dev_parse_header(skb, sll->sll_addr);
2368 	sll->sll_family = AF_PACKET;
2369 	sll->sll_hatype = dev->type;
2370 	sll->sll_protocol = skb->protocol;
2371 	sll->sll_pkttype = skb->pkt_type;
2372 	if (unlikely(po->origdev))
2373 		sll->sll_ifindex = orig_dev->ifindex;
2374 	else
2375 		sll->sll_ifindex = dev->ifindex;
2376 
2377 	smp_mb();
2378 
2379 #if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE == 1
2380 	if (po->tp_version <= TPACKET_V2) {
2381 		u8 *start, *end;
2382 
2383 		end = (u8 *) PAGE_ALIGN((unsigned long) h.raw +
2384 					macoff + snaplen);
2385 
2386 		for (start = h.raw; start < end; start += PAGE_SIZE)
2387 			flush_dcache_page(pgv_to_page(start));
2388 	}
2389 	smp_wmb();
2390 #endif
2391 
2392 	if (po->tp_version <= TPACKET_V2) {
2393 		__packet_set_status(po, h.raw, status);
2394 		sk->sk_data_ready(sk);
2395 	} else {
2396 		prb_clear_blk_fill_status(&po->rx_ring);
2397 	}
2398 
2399 drop_n_restore:
2400 	if (skb_head != skb->data && skb_shared(skb)) {
2401 		skb->data = skb_head;
2402 		skb->len = skb_len;
2403 	}
2404 drop:
2405 	if (!is_drop_n_account)
2406 		consume_skb(skb);
2407 	else
2408 		kfree_skb(skb);
2409 	return 0;
2410 
2411 drop_n_account:
2412 	is_drop_n_account = true;
2413 	po->stats.stats1.tp_drops++;
2414 	spin_unlock(&sk->sk_receive_queue.lock);
2415 
2416 	sk->sk_data_ready(sk);
2417 	kfree_skb(copy_skb);
2418 	goto drop_n_restore;
2419 }
2420 
2421 static void tpacket_destruct_skb(struct sk_buff *skb)
2422 {
2423 	struct packet_sock *po = pkt_sk(skb->sk);
2424 
2425 	if (likely(po->tx_ring.pg_vec)) {
2426 		void *ph;
2427 		__u32 ts;
2428 
2429 		ph = skb_shinfo(skb)->destructor_arg;
2430 		packet_dec_pending(&po->tx_ring);
2431 
2432 		ts = __packet_set_timestamp(po, ph, skb);
2433 		__packet_set_status(po, ph, TP_STATUS_AVAILABLE | ts);
2434 	}
2435 
2436 	sock_wfree(skb);
2437 }
2438 
2439 static void tpacket_set_protocol(const struct net_device *dev,
2440 				 struct sk_buff *skb)
2441 {
2442 	if (dev->type == ARPHRD_ETHER) {
2443 		skb_reset_mac_header(skb);
2444 		skb->protocol = eth_hdr(skb)->h_proto;
2445 	}
2446 }
2447 
2448 static int __packet_snd_vnet_parse(struct virtio_net_hdr *vnet_hdr, size_t len)
2449 {
2450 	if ((vnet_hdr->flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) &&
2451 	    (__virtio16_to_cpu(vio_le(), vnet_hdr->csum_start) +
2452 	     __virtio16_to_cpu(vio_le(), vnet_hdr->csum_offset) + 2 >
2453 	      __virtio16_to_cpu(vio_le(), vnet_hdr->hdr_len)))
2454 		vnet_hdr->hdr_len = __cpu_to_virtio16(vio_le(),
2455 			 __virtio16_to_cpu(vio_le(), vnet_hdr->csum_start) +
2456 			__virtio16_to_cpu(vio_le(), vnet_hdr->csum_offset) + 2);
2457 
2458 	if (__virtio16_to_cpu(vio_le(), vnet_hdr->hdr_len) > len)
2459 		return -EINVAL;
2460 
2461 	return 0;
2462 }
2463 
2464 static int packet_snd_vnet_parse(struct msghdr *msg, size_t *len,
2465 				 struct virtio_net_hdr *vnet_hdr)
2466 {
2467 	if (*len < sizeof(*vnet_hdr))
2468 		return -EINVAL;
2469 	*len -= sizeof(*vnet_hdr);
2470 
2471 	if (!copy_from_iter_full(vnet_hdr, sizeof(*vnet_hdr), &msg->msg_iter))
2472 		return -EFAULT;
2473 
2474 	return __packet_snd_vnet_parse(vnet_hdr, *len);
2475 }
2476 
2477 static int tpacket_fill_skb(struct packet_sock *po, struct sk_buff *skb,
2478 		void *frame, struct net_device *dev, void *data, int tp_len,
2479 		__be16 proto, unsigned char *addr, int hlen, int copylen,
2480 		const struct sockcm_cookie *sockc)
2481 {
2482 	union tpacket_uhdr ph;
2483 	int to_write, offset, len, nr_frags, len_max;
2484 	struct socket *sock = po->sk.sk_socket;
2485 	struct page *page;
2486 	int err;
2487 
2488 	ph.raw = frame;
2489 
2490 	skb->protocol = proto;
2491 	skb->dev = dev;
2492 	skb->priority = po->sk.sk_priority;
2493 	skb->mark = po->sk.sk_mark;
2494 	sock_tx_timestamp(&po->sk, sockc->tsflags, &skb_shinfo(skb)->tx_flags);
2495 	skb_shinfo(skb)->destructor_arg = ph.raw;
2496 
2497 	skb_reserve(skb, hlen);
2498 	skb_reset_network_header(skb);
2499 
2500 	to_write = tp_len;
2501 
2502 	if (sock->type == SOCK_DGRAM) {
2503 		err = dev_hard_header(skb, dev, ntohs(proto), addr,
2504 				NULL, tp_len);
2505 		if (unlikely(err < 0))
2506 			return -EINVAL;
2507 	} else if (copylen) {
2508 		int hdrlen = min_t(int, copylen, tp_len);
2509 
2510 		skb_push(skb, dev->hard_header_len);
2511 		skb_put(skb, copylen - dev->hard_header_len);
2512 		err = skb_store_bits(skb, 0, data, hdrlen);
2513 		if (unlikely(err))
2514 			return err;
2515 		if (!dev_validate_header(dev, skb->data, hdrlen))
2516 			return -EINVAL;
2517 		if (!skb->protocol)
2518 			tpacket_set_protocol(dev, skb);
2519 
2520 		data += hdrlen;
2521 		to_write -= hdrlen;
2522 	}
2523 
2524 	offset = offset_in_page(data);
2525 	len_max = PAGE_SIZE - offset;
2526 	len = ((to_write > len_max) ? len_max : to_write);
2527 
2528 	skb->data_len = to_write;
2529 	skb->len += to_write;
2530 	skb->truesize += to_write;
2531 	refcount_add(to_write, &po->sk.sk_wmem_alloc);
2532 
2533 	while (likely(to_write)) {
2534 		nr_frags = skb_shinfo(skb)->nr_frags;
2535 
2536 		if (unlikely(nr_frags >= MAX_SKB_FRAGS)) {
2537 			pr_err("Packet exceed the number of skb frags(%lu)\n",
2538 			       MAX_SKB_FRAGS);
2539 			return -EFAULT;
2540 		}
2541 
2542 		page = pgv_to_page(data);
2543 		data += len;
2544 		flush_dcache_page(page);
2545 		get_page(page);
2546 		skb_fill_page_desc(skb, nr_frags, page, offset, len);
2547 		to_write -= len;
2548 		offset = 0;
2549 		len_max = PAGE_SIZE;
2550 		len = ((to_write > len_max) ? len_max : to_write);
2551 	}
2552 
2553 	skb_probe_transport_header(skb, 0);
2554 
2555 	return tp_len;
2556 }
2557 
2558 static int tpacket_parse_header(struct packet_sock *po, void *frame,
2559 				int size_max, void **data)
2560 {
2561 	union tpacket_uhdr ph;
2562 	int tp_len, off;
2563 
2564 	ph.raw = frame;
2565 
2566 	switch (po->tp_version) {
2567 	case TPACKET_V3:
2568 		if (ph.h3->tp_next_offset != 0) {
2569 			pr_warn_once("variable sized slot not supported");
2570 			return -EINVAL;
2571 		}
2572 		tp_len = ph.h3->tp_len;
2573 		break;
2574 	case TPACKET_V2:
2575 		tp_len = ph.h2->tp_len;
2576 		break;
2577 	default:
2578 		tp_len = ph.h1->tp_len;
2579 		break;
2580 	}
2581 	if (unlikely(tp_len > size_max)) {
2582 		pr_err("packet size is too long (%d > %d)\n", tp_len, size_max);
2583 		return -EMSGSIZE;
2584 	}
2585 
2586 	if (unlikely(po->tp_tx_has_off)) {
2587 		int off_min, off_max;
2588 
2589 		off_min = po->tp_hdrlen - sizeof(struct sockaddr_ll);
2590 		off_max = po->tx_ring.frame_size - tp_len;
2591 		if (po->sk.sk_type == SOCK_DGRAM) {
2592 			switch (po->tp_version) {
2593 			case TPACKET_V3:
2594 				off = ph.h3->tp_net;
2595 				break;
2596 			case TPACKET_V2:
2597 				off = ph.h2->tp_net;
2598 				break;
2599 			default:
2600 				off = ph.h1->tp_net;
2601 				break;
2602 			}
2603 		} else {
2604 			switch (po->tp_version) {
2605 			case TPACKET_V3:
2606 				off = ph.h3->tp_mac;
2607 				break;
2608 			case TPACKET_V2:
2609 				off = ph.h2->tp_mac;
2610 				break;
2611 			default:
2612 				off = ph.h1->tp_mac;
2613 				break;
2614 			}
2615 		}
2616 		if (unlikely((off < off_min) || (off_max < off)))
2617 			return -EINVAL;
2618 	} else {
2619 		off = po->tp_hdrlen - sizeof(struct sockaddr_ll);
2620 	}
2621 
2622 	*data = frame + off;
2623 	return tp_len;
2624 }
2625 
2626 static int tpacket_snd(struct packet_sock *po, struct msghdr *msg)
2627 {
2628 	struct sk_buff *skb;
2629 	struct net_device *dev;
2630 	struct virtio_net_hdr *vnet_hdr = NULL;
2631 	struct sockcm_cookie sockc;
2632 	__be16 proto;
2633 	int err, reserve = 0;
2634 	void *ph;
2635 	DECLARE_SOCKADDR(struct sockaddr_ll *, saddr, msg->msg_name);
2636 	bool need_wait = !(msg->msg_flags & MSG_DONTWAIT);
2637 	int tp_len, size_max;
2638 	unsigned char *addr;
2639 	void *data;
2640 	int len_sum = 0;
2641 	int status = TP_STATUS_AVAILABLE;
2642 	int hlen, tlen, copylen = 0;
2643 
2644 	mutex_lock(&po->pg_vec_lock);
2645 
2646 	if (likely(saddr == NULL)) {
2647 		dev	= packet_cached_dev_get(po);
2648 		proto	= po->num;
2649 		addr	= NULL;
2650 	} else {
2651 		err = -EINVAL;
2652 		if (msg->msg_namelen < sizeof(struct sockaddr_ll))
2653 			goto out;
2654 		if (msg->msg_namelen < (saddr->sll_halen
2655 					+ offsetof(struct sockaddr_ll,
2656 						sll_addr)))
2657 			goto out;
2658 		proto	= saddr->sll_protocol;
2659 		addr	= saddr->sll_addr;
2660 		dev = dev_get_by_index(sock_net(&po->sk), saddr->sll_ifindex);
2661 	}
2662 
2663 	err = -ENXIO;
2664 	if (unlikely(dev == NULL))
2665 		goto out;
2666 	err = -ENETDOWN;
2667 	if (unlikely(!(dev->flags & IFF_UP)))
2668 		goto out_put;
2669 
2670 	sockc.tsflags = po->sk.sk_tsflags;
2671 	if (msg->msg_controllen) {
2672 		err = sock_cmsg_send(&po->sk, msg, &sockc);
2673 		if (unlikely(err))
2674 			goto out_put;
2675 	}
2676 
2677 	if (po->sk.sk_socket->type == SOCK_RAW)
2678 		reserve = dev->hard_header_len;
2679 	size_max = po->tx_ring.frame_size
2680 		- (po->tp_hdrlen - sizeof(struct sockaddr_ll));
2681 
2682 	if ((size_max > dev->mtu + reserve + VLAN_HLEN) && !po->has_vnet_hdr)
2683 		size_max = dev->mtu + reserve + VLAN_HLEN;
2684 
2685 	do {
2686 		ph = packet_current_frame(po, &po->tx_ring,
2687 					  TP_STATUS_SEND_REQUEST);
2688 		if (unlikely(ph == NULL)) {
2689 			if (need_wait && need_resched())
2690 				schedule();
2691 			continue;
2692 		}
2693 
2694 		skb = NULL;
2695 		tp_len = tpacket_parse_header(po, ph, size_max, &data);
2696 		if (tp_len < 0)
2697 			goto tpacket_error;
2698 
2699 		status = TP_STATUS_SEND_REQUEST;
2700 		hlen = LL_RESERVED_SPACE(dev);
2701 		tlen = dev->needed_tailroom;
2702 		if (po->has_vnet_hdr) {
2703 			vnet_hdr = data;
2704 			data += sizeof(*vnet_hdr);
2705 			tp_len -= sizeof(*vnet_hdr);
2706 			if (tp_len < 0 ||
2707 			    __packet_snd_vnet_parse(vnet_hdr, tp_len)) {
2708 				tp_len = -EINVAL;
2709 				goto tpacket_error;
2710 			}
2711 			copylen = __virtio16_to_cpu(vio_le(),
2712 						    vnet_hdr->hdr_len);
2713 		}
2714 		copylen = max_t(int, copylen, dev->hard_header_len);
2715 		skb = sock_alloc_send_skb(&po->sk,
2716 				hlen + tlen + sizeof(struct sockaddr_ll) +
2717 				(copylen - dev->hard_header_len),
2718 				!need_wait, &err);
2719 
2720 		if (unlikely(skb == NULL)) {
2721 			/* we assume the socket was initially writeable ... */
2722 			if (likely(len_sum > 0))
2723 				err = len_sum;
2724 			goto out_status;
2725 		}
2726 		tp_len = tpacket_fill_skb(po, skb, ph, dev, data, tp_len, proto,
2727 					  addr, hlen, copylen, &sockc);
2728 		if (likely(tp_len >= 0) &&
2729 		    tp_len > dev->mtu + reserve &&
2730 		    !po->has_vnet_hdr &&
2731 		    !packet_extra_vlan_len_allowed(dev, skb))
2732 			tp_len = -EMSGSIZE;
2733 
2734 		if (unlikely(tp_len < 0)) {
2735 tpacket_error:
2736 			if (po->tp_loss) {
2737 				__packet_set_status(po, ph,
2738 						TP_STATUS_AVAILABLE);
2739 				packet_increment_head(&po->tx_ring);
2740 				kfree_skb(skb);
2741 				continue;
2742 			} else {
2743 				status = TP_STATUS_WRONG_FORMAT;
2744 				err = tp_len;
2745 				goto out_status;
2746 			}
2747 		}
2748 
2749 		if (po->has_vnet_hdr && virtio_net_hdr_to_skb(skb, vnet_hdr,
2750 							      vio_le())) {
2751 			tp_len = -EINVAL;
2752 			goto tpacket_error;
2753 		}
2754 
2755 		skb->destructor = tpacket_destruct_skb;
2756 		__packet_set_status(po, ph, TP_STATUS_SENDING);
2757 		packet_inc_pending(&po->tx_ring);
2758 
2759 		status = TP_STATUS_SEND_REQUEST;
2760 		err = po->xmit(skb);
2761 		if (unlikely(err > 0)) {
2762 			err = net_xmit_errno(err);
2763 			if (err && __packet_get_status(po, ph) ==
2764 				   TP_STATUS_AVAILABLE) {
2765 				/* skb was destructed already */
2766 				skb = NULL;
2767 				goto out_status;
2768 			}
2769 			/*
2770 			 * skb was dropped but not destructed yet;
2771 			 * let's treat it like congestion or err < 0
2772 			 */
2773 			err = 0;
2774 		}
2775 		packet_increment_head(&po->tx_ring);
2776 		len_sum += tp_len;
2777 	} while (likely((ph != NULL) ||
2778 		/* Note: packet_read_pending() might be slow if we have
2779 		 * to call it as it's per_cpu variable, but in fast-path
2780 		 * we already short-circuit the loop with the first
2781 		 * condition, and luckily don't have to go that path
2782 		 * anyway.
2783 		 */
2784 		 (need_wait && packet_read_pending(&po->tx_ring))));
2785 
2786 	err = len_sum;
2787 	goto out_put;
2788 
2789 out_status:
2790 	__packet_set_status(po, ph, status);
2791 	kfree_skb(skb);
2792 out_put:
2793 	dev_put(dev);
2794 out:
2795 	mutex_unlock(&po->pg_vec_lock);
2796 	return err;
2797 }
2798 
2799 static struct sk_buff *packet_alloc_skb(struct sock *sk, size_t prepad,
2800 				        size_t reserve, size_t len,
2801 				        size_t linear, int noblock,
2802 				        int *err)
2803 {
2804 	struct sk_buff *skb;
2805 
2806 	/* Under a page?  Don't bother with paged skb. */
2807 	if (prepad + len < PAGE_SIZE || !linear)
2808 		linear = len;
2809 
2810 	skb = sock_alloc_send_pskb(sk, prepad + linear, len - linear, noblock,
2811 				   err, 0);
2812 	if (!skb)
2813 		return NULL;
2814 
2815 	skb_reserve(skb, reserve);
2816 	skb_put(skb, linear);
2817 	skb->data_len = len - linear;
2818 	skb->len += len - linear;
2819 
2820 	return skb;
2821 }
2822 
2823 static int packet_snd(struct socket *sock, struct msghdr *msg, size_t len)
2824 {
2825 	struct sock *sk = sock->sk;
2826 	DECLARE_SOCKADDR(struct sockaddr_ll *, saddr, msg->msg_name);
2827 	struct sk_buff *skb;
2828 	struct net_device *dev;
2829 	__be16 proto;
2830 	unsigned char *addr;
2831 	int err, reserve = 0;
2832 	struct sockcm_cookie sockc;
2833 	struct virtio_net_hdr vnet_hdr = { 0 };
2834 	int offset = 0;
2835 	struct packet_sock *po = pkt_sk(sk);
2836 	bool has_vnet_hdr = false;
2837 	int hlen, tlen, linear;
2838 	int extra_len = 0;
2839 
2840 	/*
2841 	 *	Get and verify the address.
2842 	 */
2843 
2844 	if (likely(saddr == NULL)) {
2845 		dev	= packet_cached_dev_get(po);
2846 		proto	= po->num;
2847 		addr	= NULL;
2848 	} else {
2849 		err = -EINVAL;
2850 		if (msg->msg_namelen < sizeof(struct sockaddr_ll))
2851 			goto out;
2852 		if (msg->msg_namelen < (saddr->sll_halen + offsetof(struct sockaddr_ll, sll_addr)))
2853 			goto out;
2854 		proto	= saddr->sll_protocol;
2855 		addr	= saddr->sll_addr;
2856 		dev = dev_get_by_index(sock_net(sk), saddr->sll_ifindex);
2857 	}
2858 
2859 	err = -ENXIO;
2860 	if (unlikely(dev == NULL))
2861 		goto out_unlock;
2862 	err = -ENETDOWN;
2863 	if (unlikely(!(dev->flags & IFF_UP)))
2864 		goto out_unlock;
2865 
2866 	sockc.tsflags = sk->sk_tsflags;
2867 	sockc.mark = sk->sk_mark;
2868 	if (msg->msg_controllen) {
2869 		err = sock_cmsg_send(sk, msg, &sockc);
2870 		if (unlikely(err))
2871 			goto out_unlock;
2872 	}
2873 
2874 	if (sock->type == SOCK_RAW)
2875 		reserve = dev->hard_header_len;
2876 	if (po->has_vnet_hdr) {
2877 		err = packet_snd_vnet_parse(msg, &len, &vnet_hdr);
2878 		if (err)
2879 			goto out_unlock;
2880 		has_vnet_hdr = true;
2881 	}
2882 
2883 	if (unlikely(sock_flag(sk, SOCK_NOFCS))) {
2884 		if (!netif_supports_nofcs(dev)) {
2885 			err = -EPROTONOSUPPORT;
2886 			goto out_unlock;
2887 		}
2888 		extra_len = 4; /* We're doing our own CRC */
2889 	}
2890 
2891 	err = -EMSGSIZE;
2892 	if (!vnet_hdr.gso_type &&
2893 	    (len > dev->mtu + reserve + VLAN_HLEN + extra_len))
2894 		goto out_unlock;
2895 
2896 	err = -ENOBUFS;
2897 	hlen = LL_RESERVED_SPACE(dev);
2898 	tlen = dev->needed_tailroom;
2899 	linear = __virtio16_to_cpu(vio_le(), vnet_hdr.hdr_len);
2900 	linear = max(linear, min_t(int, len, dev->hard_header_len));
2901 	skb = packet_alloc_skb(sk, hlen + tlen, hlen, len, linear,
2902 			       msg->msg_flags & MSG_DONTWAIT, &err);
2903 	if (skb == NULL)
2904 		goto out_unlock;
2905 
2906 	skb_set_network_header(skb, reserve);
2907 
2908 	err = -EINVAL;
2909 	if (sock->type == SOCK_DGRAM) {
2910 		offset = dev_hard_header(skb, dev, ntohs(proto), addr, NULL, len);
2911 		if (unlikely(offset < 0))
2912 			goto out_free;
2913 	}
2914 
2915 	/* Returns -EFAULT on error */
2916 	err = skb_copy_datagram_from_iter(skb, offset, &msg->msg_iter, len);
2917 	if (err)
2918 		goto out_free;
2919 
2920 	if (sock->type == SOCK_RAW &&
2921 	    !dev_validate_header(dev, skb->data, len)) {
2922 		err = -EINVAL;
2923 		goto out_free;
2924 	}
2925 
2926 	sock_tx_timestamp(sk, sockc.tsflags, &skb_shinfo(skb)->tx_flags);
2927 
2928 	if (!vnet_hdr.gso_type && (len > dev->mtu + reserve + extra_len) &&
2929 	    !packet_extra_vlan_len_allowed(dev, skb)) {
2930 		err = -EMSGSIZE;
2931 		goto out_free;
2932 	}
2933 
2934 	skb->protocol = proto;
2935 	skb->dev = dev;
2936 	skb->priority = sk->sk_priority;
2937 	skb->mark = sockc.mark;
2938 
2939 	if (has_vnet_hdr) {
2940 		err = virtio_net_hdr_to_skb(skb, &vnet_hdr, vio_le());
2941 		if (err)
2942 			goto out_free;
2943 		len += sizeof(vnet_hdr);
2944 	}
2945 
2946 	skb_probe_transport_header(skb, reserve);
2947 
2948 	if (unlikely(extra_len == 4))
2949 		skb->no_fcs = 1;
2950 
2951 	err = po->xmit(skb);
2952 	if (err > 0 && (err = net_xmit_errno(err)) != 0)
2953 		goto out_unlock;
2954 
2955 	dev_put(dev);
2956 
2957 	return len;
2958 
2959 out_free:
2960 	kfree_skb(skb);
2961 out_unlock:
2962 	if (dev)
2963 		dev_put(dev);
2964 out:
2965 	return err;
2966 }
2967 
2968 static int packet_sendmsg(struct socket *sock, struct msghdr *msg, size_t len)
2969 {
2970 	struct sock *sk = sock->sk;
2971 	struct packet_sock *po = pkt_sk(sk);
2972 
2973 	if (po->tx_ring.pg_vec)
2974 		return tpacket_snd(po, msg);
2975 	else
2976 		return packet_snd(sock, msg, len);
2977 }
2978 
2979 /*
2980  *	Close a PACKET socket. This is fairly simple. We immediately go
2981  *	to 'closed' state and remove our protocol entry in the device list.
2982  */
2983 
2984 static int packet_release(struct socket *sock)
2985 {
2986 	struct sock *sk = sock->sk;
2987 	struct packet_sock *po;
2988 	struct packet_fanout *f;
2989 	struct net *net;
2990 	union tpacket_req_u req_u;
2991 
2992 	if (!sk)
2993 		return 0;
2994 
2995 	net = sock_net(sk);
2996 	po = pkt_sk(sk);
2997 
2998 	mutex_lock(&net->packet.sklist_lock);
2999 	sk_del_node_init_rcu(sk);
3000 	mutex_unlock(&net->packet.sklist_lock);
3001 
3002 	preempt_disable();
3003 	sock_prot_inuse_add(net, sk->sk_prot, -1);
3004 	preempt_enable();
3005 
3006 	spin_lock(&po->bind_lock);
3007 	unregister_prot_hook(sk, false);
3008 	packet_cached_dev_reset(po);
3009 
3010 	if (po->prot_hook.dev) {
3011 		dev_put(po->prot_hook.dev);
3012 		po->prot_hook.dev = NULL;
3013 	}
3014 	spin_unlock(&po->bind_lock);
3015 
3016 	packet_flush_mclist(sk);
3017 
3018 	lock_sock(sk);
3019 	if (po->rx_ring.pg_vec) {
3020 		memset(&req_u, 0, sizeof(req_u));
3021 		packet_set_ring(sk, &req_u, 1, 0);
3022 	}
3023 
3024 	if (po->tx_ring.pg_vec) {
3025 		memset(&req_u, 0, sizeof(req_u));
3026 		packet_set_ring(sk, &req_u, 1, 1);
3027 	}
3028 	release_sock(sk);
3029 
3030 	f = fanout_release(sk);
3031 
3032 	synchronize_net();
3033 
3034 	if (f) {
3035 		kfree(po->rollover);
3036 		fanout_release_data(f);
3037 		kfree(f);
3038 	}
3039 	/*
3040 	 *	Now the socket is dead. No more input will appear.
3041 	 */
3042 	sock_orphan(sk);
3043 	sock->sk = NULL;
3044 
3045 	/* Purge queues */
3046 
3047 	skb_queue_purge(&sk->sk_receive_queue);
3048 	packet_free_pending(po);
3049 	sk_refcnt_debug_release(sk);
3050 
3051 	sock_put(sk);
3052 	return 0;
3053 }
3054 
3055 /*
3056  *	Attach a packet hook.
3057  */
3058 
3059 static int packet_do_bind(struct sock *sk, const char *name, int ifindex,
3060 			  __be16 proto)
3061 {
3062 	struct packet_sock *po = pkt_sk(sk);
3063 	struct net_device *dev_curr;
3064 	__be16 proto_curr;
3065 	bool need_rehook;
3066 	struct net_device *dev = NULL;
3067 	int ret = 0;
3068 	bool unlisted = false;
3069 
3070 	lock_sock(sk);
3071 	spin_lock(&po->bind_lock);
3072 	rcu_read_lock();
3073 
3074 	if (po->fanout) {
3075 		ret = -EINVAL;
3076 		goto out_unlock;
3077 	}
3078 
3079 	if (name) {
3080 		dev = dev_get_by_name_rcu(sock_net(sk), name);
3081 		if (!dev) {
3082 			ret = -ENODEV;
3083 			goto out_unlock;
3084 		}
3085 	} else if (ifindex) {
3086 		dev = dev_get_by_index_rcu(sock_net(sk), ifindex);
3087 		if (!dev) {
3088 			ret = -ENODEV;
3089 			goto out_unlock;
3090 		}
3091 	}
3092 
3093 	if (dev)
3094 		dev_hold(dev);
3095 
3096 	proto_curr = po->prot_hook.type;
3097 	dev_curr = po->prot_hook.dev;
3098 
3099 	need_rehook = proto_curr != proto || dev_curr != dev;
3100 
3101 	if (need_rehook) {
3102 		if (po->running) {
3103 			rcu_read_unlock();
3104 			/* prevents packet_notifier() from calling
3105 			 * register_prot_hook()
3106 			 */
3107 			po->num = 0;
3108 			__unregister_prot_hook(sk, true);
3109 			rcu_read_lock();
3110 			dev_curr = po->prot_hook.dev;
3111 			if (dev)
3112 				unlisted = !dev_get_by_index_rcu(sock_net(sk),
3113 								 dev->ifindex);
3114 		}
3115 
3116 		BUG_ON(po->running);
3117 		po->num = proto;
3118 		po->prot_hook.type = proto;
3119 
3120 		if (unlikely(unlisted)) {
3121 			dev_put(dev);
3122 			po->prot_hook.dev = NULL;
3123 			po->ifindex = -1;
3124 			packet_cached_dev_reset(po);
3125 		} else {
3126 			po->prot_hook.dev = dev;
3127 			po->ifindex = dev ? dev->ifindex : 0;
3128 			packet_cached_dev_assign(po, dev);
3129 		}
3130 	}
3131 	if (dev_curr)
3132 		dev_put(dev_curr);
3133 
3134 	if (proto == 0 || !need_rehook)
3135 		goto out_unlock;
3136 
3137 	if (!unlisted && (!dev || (dev->flags & IFF_UP))) {
3138 		register_prot_hook(sk);
3139 	} else {
3140 		sk->sk_err = ENETDOWN;
3141 		if (!sock_flag(sk, SOCK_DEAD))
3142 			sk->sk_error_report(sk);
3143 	}
3144 
3145 out_unlock:
3146 	rcu_read_unlock();
3147 	spin_unlock(&po->bind_lock);
3148 	release_sock(sk);
3149 	return ret;
3150 }
3151 
3152 /*
3153  *	Bind a packet socket to a device
3154  */
3155 
3156 static int packet_bind_spkt(struct socket *sock, struct sockaddr *uaddr,
3157 			    int addr_len)
3158 {
3159 	struct sock *sk = sock->sk;
3160 	char name[sizeof(uaddr->sa_data) + 1];
3161 
3162 	/*
3163 	 *	Check legality
3164 	 */
3165 
3166 	if (addr_len != sizeof(struct sockaddr))
3167 		return -EINVAL;
3168 	/* uaddr->sa_data comes from the userspace, it's not guaranteed to be
3169 	 * zero-terminated.
3170 	 */
3171 	memcpy(name, uaddr->sa_data, sizeof(uaddr->sa_data));
3172 	name[sizeof(uaddr->sa_data)] = 0;
3173 
3174 	return packet_do_bind(sk, name, 0, pkt_sk(sk)->num);
3175 }
3176 
3177 static int packet_bind(struct socket *sock, struct sockaddr *uaddr, int addr_len)
3178 {
3179 	struct sockaddr_ll *sll = (struct sockaddr_ll *)uaddr;
3180 	struct sock *sk = sock->sk;
3181 
3182 	/*
3183 	 *	Check legality
3184 	 */
3185 
3186 	if (addr_len < sizeof(struct sockaddr_ll))
3187 		return -EINVAL;
3188 	if (sll->sll_family != AF_PACKET)
3189 		return -EINVAL;
3190 
3191 	return packet_do_bind(sk, NULL, sll->sll_ifindex,
3192 			      sll->sll_protocol ? : pkt_sk(sk)->num);
3193 }
3194 
3195 static struct proto packet_proto = {
3196 	.name	  = "PACKET",
3197 	.owner	  = THIS_MODULE,
3198 	.obj_size = sizeof(struct packet_sock),
3199 };
3200 
3201 /*
3202  *	Create a packet of type SOCK_PACKET.
3203  */
3204 
3205 static int packet_create(struct net *net, struct socket *sock, int protocol,
3206 			 int kern)
3207 {
3208 	struct sock *sk;
3209 	struct packet_sock *po;
3210 	__be16 proto = (__force __be16)protocol; /* weird, but documented */
3211 	int err;
3212 
3213 	if (!ns_capable(net->user_ns, CAP_NET_RAW))
3214 		return -EPERM;
3215 	if (sock->type != SOCK_DGRAM && sock->type != SOCK_RAW &&
3216 	    sock->type != SOCK_PACKET)
3217 		return -ESOCKTNOSUPPORT;
3218 
3219 	sock->state = SS_UNCONNECTED;
3220 
3221 	err = -ENOBUFS;
3222 	sk = sk_alloc(net, PF_PACKET, GFP_KERNEL, &packet_proto, kern);
3223 	if (sk == NULL)
3224 		goto out;
3225 
3226 	sock->ops = &packet_ops;
3227 	if (sock->type == SOCK_PACKET)
3228 		sock->ops = &packet_ops_spkt;
3229 
3230 	sock_init_data(sock, sk);
3231 
3232 	po = pkt_sk(sk);
3233 	sk->sk_family = PF_PACKET;
3234 	po->num = proto;
3235 	po->xmit = dev_queue_xmit;
3236 
3237 	err = packet_alloc_pending(po);
3238 	if (err)
3239 		goto out2;
3240 
3241 	packet_cached_dev_reset(po);
3242 
3243 	sk->sk_destruct = packet_sock_destruct;
3244 	sk_refcnt_debug_inc(sk);
3245 
3246 	/*
3247 	 *	Attach a protocol block
3248 	 */
3249 
3250 	spin_lock_init(&po->bind_lock);
3251 	mutex_init(&po->pg_vec_lock);
3252 	po->rollover = NULL;
3253 	po->prot_hook.func = packet_rcv;
3254 
3255 	if (sock->type == SOCK_PACKET)
3256 		po->prot_hook.func = packet_rcv_spkt;
3257 
3258 	po->prot_hook.af_packet_priv = sk;
3259 
3260 	if (proto) {
3261 		po->prot_hook.type = proto;
3262 		__register_prot_hook(sk);
3263 	}
3264 
3265 	mutex_lock(&net->packet.sklist_lock);
3266 	sk_add_node_rcu(sk, &net->packet.sklist);
3267 	mutex_unlock(&net->packet.sklist_lock);
3268 
3269 	preempt_disable();
3270 	sock_prot_inuse_add(net, &packet_proto, 1);
3271 	preempt_enable();
3272 
3273 	return 0;
3274 out2:
3275 	sk_free(sk);
3276 out:
3277 	return err;
3278 }
3279 
3280 /*
3281  *	Pull a packet from our receive queue and hand it to the user.
3282  *	If necessary we block.
3283  */
3284 
3285 static int packet_recvmsg(struct socket *sock, struct msghdr *msg, size_t len,
3286 			  int flags)
3287 {
3288 	struct sock *sk = sock->sk;
3289 	struct sk_buff *skb;
3290 	int copied, err;
3291 	int vnet_hdr_len = 0;
3292 	unsigned int origlen = 0;
3293 
3294 	err = -EINVAL;
3295 	if (flags & ~(MSG_PEEK|MSG_DONTWAIT|MSG_TRUNC|MSG_CMSG_COMPAT|MSG_ERRQUEUE))
3296 		goto out;
3297 
3298 #if 0
3299 	/* What error should we return now? EUNATTACH? */
3300 	if (pkt_sk(sk)->ifindex < 0)
3301 		return -ENODEV;
3302 #endif
3303 
3304 	if (flags & MSG_ERRQUEUE) {
3305 		err = sock_recv_errqueue(sk, msg, len,
3306 					 SOL_PACKET, PACKET_TX_TIMESTAMP);
3307 		goto out;
3308 	}
3309 
3310 	/*
3311 	 *	Call the generic datagram receiver. This handles all sorts
3312 	 *	of horrible races and re-entrancy so we can forget about it
3313 	 *	in the protocol layers.
3314 	 *
3315 	 *	Now it will return ENETDOWN, if device have just gone down,
3316 	 *	but then it will block.
3317 	 */
3318 
3319 	skb = skb_recv_datagram(sk, flags, flags & MSG_DONTWAIT, &err);
3320 
3321 	/*
3322 	 *	An error occurred so return it. Because skb_recv_datagram()
3323 	 *	handles the blocking we don't see and worry about blocking
3324 	 *	retries.
3325 	 */
3326 
3327 	if (skb == NULL)
3328 		goto out;
3329 
3330 	if (pkt_sk(sk)->pressure)
3331 		packet_rcv_has_room(pkt_sk(sk), NULL);
3332 
3333 	if (pkt_sk(sk)->has_vnet_hdr) {
3334 		err = packet_rcv_vnet(msg, skb, &len);
3335 		if (err)
3336 			goto out_free;
3337 		vnet_hdr_len = sizeof(struct virtio_net_hdr);
3338 	}
3339 
3340 	/* You lose any data beyond the buffer you gave. If it worries
3341 	 * a user program they can ask the device for its MTU
3342 	 * anyway.
3343 	 */
3344 	copied = skb->len;
3345 	if (copied > len) {
3346 		copied = len;
3347 		msg->msg_flags |= MSG_TRUNC;
3348 	}
3349 
3350 	err = skb_copy_datagram_msg(skb, 0, msg, copied);
3351 	if (err)
3352 		goto out_free;
3353 
3354 	if (sock->type != SOCK_PACKET) {
3355 		struct sockaddr_ll *sll = &PACKET_SKB_CB(skb)->sa.ll;
3356 
3357 		/* Original length was stored in sockaddr_ll fields */
3358 		origlen = PACKET_SKB_CB(skb)->sa.origlen;
3359 		sll->sll_family = AF_PACKET;
3360 		sll->sll_protocol = skb->protocol;
3361 	}
3362 
3363 	sock_recv_ts_and_drops(msg, sk, skb);
3364 
3365 	if (msg->msg_name) {
3366 		/* If the address length field is there to be filled
3367 		 * in, we fill it in now.
3368 		 */
3369 		if (sock->type == SOCK_PACKET) {
3370 			__sockaddr_check_size(sizeof(struct sockaddr_pkt));
3371 			msg->msg_namelen = sizeof(struct sockaddr_pkt);
3372 		} else {
3373 			struct sockaddr_ll *sll = &PACKET_SKB_CB(skb)->sa.ll;
3374 
3375 			msg->msg_namelen = sll->sll_halen +
3376 				offsetof(struct sockaddr_ll, sll_addr);
3377 		}
3378 		memcpy(msg->msg_name, &PACKET_SKB_CB(skb)->sa,
3379 		       msg->msg_namelen);
3380 	}
3381 
3382 	if (pkt_sk(sk)->auxdata) {
3383 		struct tpacket_auxdata aux;
3384 
3385 		aux.tp_status = TP_STATUS_USER;
3386 		if (skb->ip_summed == CHECKSUM_PARTIAL)
3387 			aux.tp_status |= TP_STATUS_CSUMNOTREADY;
3388 		else if (skb->pkt_type != PACKET_OUTGOING &&
3389 			 (skb->ip_summed == CHECKSUM_COMPLETE ||
3390 			  skb_csum_unnecessary(skb)))
3391 			aux.tp_status |= TP_STATUS_CSUM_VALID;
3392 
3393 		aux.tp_len = origlen;
3394 		aux.tp_snaplen = skb->len;
3395 		aux.tp_mac = 0;
3396 		aux.tp_net = skb_network_offset(skb);
3397 		if (skb_vlan_tag_present(skb)) {
3398 			aux.tp_vlan_tci = skb_vlan_tag_get(skb);
3399 			aux.tp_vlan_tpid = ntohs(skb->vlan_proto);
3400 			aux.tp_status |= TP_STATUS_VLAN_VALID | TP_STATUS_VLAN_TPID_VALID;
3401 		} else {
3402 			aux.tp_vlan_tci = 0;
3403 			aux.tp_vlan_tpid = 0;
3404 		}
3405 		put_cmsg(msg, SOL_PACKET, PACKET_AUXDATA, sizeof(aux), &aux);
3406 	}
3407 
3408 	/*
3409 	 *	Free or return the buffer as appropriate. Again this
3410 	 *	hides all the races and re-entrancy issues from us.
3411 	 */
3412 	err = vnet_hdr_len + ((flags&MSG_TRUNC) ? skb->len : copied);
3413 
3414 out_free:
3415 	skb_free_datagram(sk, skb);
3416 out:
3417 	return err;
3418 }
3419 
3420 static int packet_getname_spkt(struct socket *sock, struct sockaddr *uaddr,
3421 			       int peer)
3422 {
3423 	struct net_device *dev;
3424 	struct sock *sk	= sock->sk;
3425 
3426 	if (peer)
3427 		return -EOPNOTSUPP;
3428 
3429 	uaddr->sa_family = AF_PACKET;
3430 	memset(uaddr->sa_data, 0, sizeof(uaddr->sa_data));
3431 	rcu_read_lock();
3432 	dev = dev_get_by_index_rcu(sock_net(sk), pkt_sk(sk)->ifindex);
3433 	if (dev)
3434 		strlcpy(uaddr->sa_data, dev->name, sizeof(uaddr->sa_data));
3435 	rcu_read_unlock();
3436 
3437 	return sizeof(*uaddr);
3438 }
3439 
3440 static int packet_getname(struct socket *sock, struct sockaddr *uaddr,
3441 			  int peer)
3442 {
3443 	struct net_device *dev;
3444 	struct sock *sk = sock->sk;
3445 	struct packet_sock *po = pkt_sk(sk);
3446 	DECLARE_SOCKADDR(struct sockaddr_ll *, sll, uaddr);
3447 
3448 	if (peer)
3449 		return -EOPNOTSUPP;
3450 
3451 	sll->sll_family = AF_PACKET;
3452 	sll->sll_ifindex = po->ifindex;
3453 	sll->sll_protocol = po->num;
3454 	sll->sll_pkttype = 0;
3455 	rcu_read_lock();
3456 	dev = dev_get_by_index_rcu(sock_net(sk), po->ifindex);
3457 	if (dev) {
3458 		sll->sll_hatype = dev->type;
3459 		sll->sll_halen = dev->addr_len;
3460 		memcpy(sll->sll_addr, dev->dev_addr, dev->addr_len);
3461 	} else {
3462 		sll->sll_hatype = 0;	/* Bad: we have no ARPHRD_UNSPEC */
3463 		sll->sll_halen = 0;
3464 	}
3465 	rcu_read_unlock();
3466 
3467 	return offsetof(struct sockaddr_ll, sll_addr) + sll->sll_halen;
3468 }
3469 
3470 static int packet_dev_mc(struct net_device *dev, struct packet_mclist *i,
3471 			 int what)
3472 {
3473 	switch (i->type) {
3474 	case PACKET_MR_MULTICAST:
3475 		if (i->alen != dev->addr_len)
3476 			return -EINVAL;
3477 		if (what > 0)
3478 			return dev_mc_add(dev, i->addr);
3479 		else
3480 			return dev_mc_del(dev, i->addr);
3481 		break;
3482 	case PACKET_MR_PROMISC:
3483 		return dev_set_promiscuity(dev, what);
3484 	case PACKET_MR_ALLMULTI:
3485 		return dev_set_allmulti(dev, what);
3486 	case PACKET_MR_UNICAST:
3487 		if (i->alen != dev->addr_len)
3488 			return -EINVAL;
3489 		if (what > 0)
3490 			return dev_uc_add(dev, i->addr);
3491 		else
3492 			return dev_uc_del(dev, i->addr);
3493 		break;
3494 	default:
3495 		break;
3496 	}
3497 	return 0;
3498 }
3499 
3500 static void packet_dev_mclist_delete(struct net_device *dev,
3501 				     struct packet_mclist **mlp)
3502 {
3503 	struct packet_mclist *ml;
3504 
3505 	while ((ml = *mlp) != NULL) {
3506 		if (ml->ifindex == dev->ifindex) {
3507 			packet_dev_mc(dev, ml, -1);
3508 			*mlp = ml->next;
3509 			kfree(ml);
3510 		} else
3511 			mlp = &ml->next;
3512 	}
3513 }
3514 
3515 static int packet_mc_add(struct sock *sk, struct packet_mreq_max *mreq)
3516 {
3517 	struct packet_sock *po = pkt_sk(sk);
3518 	struct packet_mclist *ml, *i;
3519 	struct net_device *dev;
3520 	int err;
3521 
3522 	rtnl_lock();
3523 
3524 	err = -ENODEV;
3525 	dev = __dev_get_by_index(sock_net(sk), mreq->mr_ifindex);
3526 	if (!dev)
3527 		goto done;
3528 
3529 	err = -EINVAL;
3530 	if (mreq->mr_alen > dev->addr_len)
3531 		goto done;
3532 
3533 	err = -ENOBUFS;
3534 	i = kmalloc(sizeof(*i), GFP_KERNEL);
3535 	if (i == NULL)
3536 		goto done;
3537 
3538 	err = 0;
3539 	for (ml = po->mclist; ml; ml = ml->next) {
3540 		if (ml->ifindex == mreq->mr_ifindex &&
3541 		    ml->type == mreq->mr_type &&
3542 		    ml->alen == mreq->mr_alen &&
3543 		    memcmp(ml->addr, mreq->mr_address, ml->alen) == 0) {
3544 			ml->count++;
3545 			/* Free the new element ... */
3546 			kfree(i);
3547 			goto done;
3548 		}
3549 	}
3550 
3551 	i->type = mreq->mr_type;
3552 	i->ifindex = mreq->mr_ifindex;
3553 	i->alen = mreq->mr_alen;
3554 	memcpy(i->addr, mreq->mr_address, i->alen);
3555 	memset(i->addr + i->alen, 0, sizeof(i->addr) - i->alen);
3556 	i->count = 1;
3557 	i->next = po->mclist;
3558 	po->mclist = i;
3559 	err = packet_dev_mc(dev, i, 1);
3560 	if (err) {
3561 		po->mclist = i->next;
3562 		kfree(i);
3563 	}
3564 
3565 done:
3566 	rtnl_unlock();
3567 	return err;
3568 }
3569 
3570 static int packet_mc_drop(struct sock *sk, struct packet_mreq_max *mreq)
3571 {
3572 	struct packet_mclist *ml, **mlp;
3573 
3574 	rtnl_lock();
3575 
3576 	for (mlp = &pkt_sk(sk)->mclist; (ml = *mlp) != NULL; mlp = &ml->next) {
3577 		if (ml->ifindex == mreq->mr_ifindex &&
3578 		    ml->type == mreq->mr_type &&
3579 		    ml->alen == mreq->mr_alen &&
3580 		    memcmp(ml->addr, mreq->mr_address, ml->alen) == 0) {
3581 			if (--ml->count == 0) {
3582 				struct net_device *dev;
3583 				*mlp = ml->next;
3584 				dev = __dev_get_by_index(sock_net(sk), ml->ifindex);
3585 				if (dev)
3586 					packet_dev_mc(dev, ml, -1);
3587 				kfree(ml);
3588 			}
3589 			break;
3590 		}
3591 	}
3592 	rtnl_unlock();
3593 	return 0;
3594 }
3595 
3596 static void packet_flush_mclist(struct sock *sk)
3597 {
3598 	struct packet_sock *po = pkt_sk(sk);
3599 	struct packet_mclist *ml;
3600 
3601 	if (!po->mclist)
3602 		return;
3603 
3604 	rtnl_lock();
3605 	while ((ml = po->mclist) != NULL) {
3606 		struct net_device *dev;
3607 
3608 		po->mclist = ml->next;
3609 		dev = __dev_get_by_index(sock_net(sk), ml->ifindex);
3610 		if (dev != NULL)
3611 			packet_dev_mc(dev, ml, -1);
3612 		kfree(ml);
3613 	}
3614 	rtnl_unlock();
3615 }
3616 
3617 static int
3618 packet_setsockopt(struct socket *sock, int level, int optname, char __user *optval, unsigned int optlen)
3619 {
3620 	struct sock *sk = sock->sk;
3621 	struct packet_sock *po = pkt_sk(sk);
3622 	int ret;
3623 
3624 	if (level != SOL_PACKET)
3625 		return -ENOPROTOOPT;
3626 
3627 	switch (optname) {
3628 	case PACKET_ADD_MEMBERSHIP:
3629 	case PACKET_DROP_MEMBERSHIP:
3630 	{
3631 		struct packet_mreq_max mreq;
3632 		int len = optlen;
3633 		memset(&mreq, 0, sizeof(mreq));
3634 		if (len < sizeof(struct packet_mreq))
3635 			return -EINVAL;
3636 		if (len > sizeof(mreq))
3637 			len = sizeof(mreq);
3638 		if (copy_from_user(&mreq, optval, len))
3639 			return -EFAULT;
3640 		if (len < (mreq.mr_alen + offsetof(struct packet_mreq, mr_address)))
3641 			return -EINVAL;
3642 		if (optname == PACKET_ADD_MEMBERSHIP)
3643 			ret = packet_mc_add(sk, &mreq);
3644 		else
3645 			ret = packet_mc_drop(sk, &mreq);
3646 		return ret;
3647 	}
3648 
3649 	case PACKET_RX_RING:
3650 	case PACKET_TX_RING:
3651 	{
3652 		union tpacket_req_u req_u;
3653 		int len;
3654 
3655 		lock_sock(sk);
3656 		switch (po->tp_version) {
3657 		case TPACKET_V1:
3658 		case TPACKET_V2:
3659 			len = sizeof(req_u.req);
3660 			break;
3661 		case TPACKET_V3:
3662 		default:
3663 			len = sizeof(req_u.req3);
3664 			break;
3665 		}
3666 		if (optlen < len) {
3667 			ret = -EINVAL;
3668 		} else {
3669 			if (copy_from_user(&req_u.req, optval, len))
3670 				ret = -EFAULT;
3671 			else
3672 				ret = packet_set_ring(sk, &req_u, 0,
3673 						    optname == PACKET_TX_RING);
3674 		}
3675 		release_sock(sk);
3676 		return ret;
3677 	}
3678 	case PACKET_COPY_THRESH:
3679 	{
3680 		int val;
3681 
3682 		if (optlen != sizeof(val))
3683 			return -EINVAL;
3684 		if (copy_from_user(&val, optval, sizeof(val)))
3685 			return -EFAULT;
3686 
3687 		pkt_sk(sk)->copy_thresh = val;
3688 		return 0;
3689 	}
3690 	case PACKET_VERSION:
3691 	{
3692 		int val;
3693 
3694 		if (optlen != sizeof(val))
3695 			return -EINVAL;
3696 		if (copy_from_user(&val, optval, sizeof(val)))
3697 			return -EFAULT;
3698 		switch (val) {
3699 		case TPACKET_V1:
3700 		case TPACKET_V2:
3701 		case TPACKET_V3:
3702 			break;
3703 		default:
3704 			return -EINVAL;
3705 		}
3706 		lock_sock(sk);
3707 		if (po->rx_ring.pg_vec || po->tx_ring.pg_vec) {
3708 			ret = -EBUSY;
3709 		} else {
3710 			po->tp_version = val;
3711 			ret = 0;
3712 		}
3713 		release_sock(sk);
3714 		return ret;
3715 	}
3716 	case PACKET_RESERVE:
3717 	{
3718 		unsigned int val;
3719 
3720 		if (optlen != sizeof(val))
3721 			return -EINVAL;
3722 		if (copy_from_user(&val, optval, sizeof(val)))
3723 			return -EFAULT;
3724 		if (val > INT_MAX)
3725 			return -EINVAL;
3726 		lock_sock(sk);
3727 		if (po->rx_ring.pg_vec || po->tx_ring.pg_vec) {
3728 			ret = -EBUSY;
3729 		} else {
3730 			po->tp_reserve = val;
3731 			ret = 0;
3732 		}
3733 		release_sock(sk);
3734 		return ret;
3735 	}
3736 	case PACKET_LOSS:
3737 	{
3738 		unsigned int val;
3739 
3740 		if (optlen != sizeof(val))
3741 			return -EINVAL;
3742 		if (copy_from_user(&val, optval, sizeof(val)))
3743 			return -EFAULT;
3744 
3745 		lock_sock(sk);
3746 		if (po->rx_ring.pg_vec || po->tx_ring.pg_vec) {
3747 			ret = -EBUSY;
3748 		} else {
3749 			po->tp_loss = !!val;
3750 			ret = 0;
3751 		}
3752 		release_sock(sk);
3753 		return ret;
3754 	}
3755 	case PACKET_AUXDATA:
3756 	{
3757 		int val;
3758 
3759 		if (optlen < sizeof(val))
3760 			return -EINVAL;
3761 		if (copy_from_user(&val, optval, sizeof(val)))
3762 			return -EFAULT;
3763 
3764 		lock_sock(sk);
3765 		po->auxdata = !!val;
3766 		release_sock(sk);
3767 		return 0;
3768 	}
3769 	case PACKET_ORIGDEV:
3770 	{
3771 		int val;
3772 
3773 		if (optlen < sizeof(val))
3774 			return -EINVAL;
3775 		if (copy_from_user(&val, optval, sizeof(val)))
3776 			return -EFAULT;
3777 
3778 		lock_sock(sk);
3779 		po->origdev = !!val;
3780 		release_sock(sk);
3781 		return 0;
3782 	}
3783 	case PACKET_VNET_HDR:
3784 	{
3785 		int val;
3786 
3787 		if (sock->type != SOCK_RAW)
3788 			return -EINVAL;
3789 		if (optlen < sizeof(val))
3790 			return -EINVAL;
3791 		if (copy_from_user(&val, optval, sizeof(val)))
3792 			return -EFAULT;
3793 
3794 		lock_sock(sk);
3795 		if (po->rx_ring.pg_vec || po->tx_ring.pg_vec) {
3796 			ret = -EBUSY;
3797 		} else {
3798 			po->has_vnet_hdr = !!val;
3799 			ret = 0;
3800 		}
3801 		release_sock(sk);
3802 		return ret;
3803 	}
3804 	case PACKET_TIMESTAMP:
3805 	{
3806 		int val;
3807 
3808 		if (optlen != sizeof(val))
3809 			return -EINVAL;
3810 		if (copy_from_user(&val, optval, sizeof(val)))
3811 			return -EFAULT;
3812 
3813 		po->tp_tstamp = val;
3814 		return 0;
3815 	}
3816 	case PACKET_FANOUT:
3817 	{
3818 		int val;
3819 
3820 		if (optlen != sizeof(val))
3821 			return -EINVAL;
3822 		if (copy_from_user(&val, optval, sizeof(val)))
3823 			return -EFAULT;
3824 
3825 		return fanout_add(sk, val & 0xffff, val >> 16);
3826 	}
3827 	case PACKET_FANOUT_DATA:
3828 	{
3829 		if (!po->fanout)
3830 			return -EINVAL;
3831 
3832 		return fanout_set_data(po, optval, optlen);
3833 	}
3834 	case PACKET_TX_HAS_OFF:
3835 	{
3836 		unsigned int val;
3837 
3838 		if (optlen != sizeof(val))
3839 			return -EINVAL;
3840 		if (copy_from_user(&val, optval, sizeof(val)))
3841 			return -EFAULT;
3842 
3843 		lock_sock(sk);
3844 		if (po->rx_ring.pg_vec || po->tx_ring.pg_vec) {
3845 			ret = -EBUSY;
3846 		} else {
3847 			po->tp_tx_has_off = !!val;
3848 			ret = 0;
3849 		}
3850 		release_sock(sk);
3851 		return 0;
3852 	}
3853 	case PACKET_QDISC_BYPASS:
3854 	{
3855 		int val;
3856 
3857 		if (optlen != sizeof(val))
3858 			return -EINVAL;
3859 		if (copy_from_user(&val, optval, sizeof(val)))
3860 			return -EFAULT;
3861 
3862 		po->xmit = val ? packet_direct_xmit : dev_queue_xmit;
3863 		return 0;
3864 	}
3865 	default:
3866 		return -ENOPROTOOPT;
3867 	}
3868 }
3869 
3870 static int packet_getsockopt(struct socket *sock, int level, int optname,
3871 			     char __user *optval, int __user *optlen)
3872 {
3873 	int len;
3874 	int val, lv = sizeof(val);
3875 	struct sock *sk = sock->sk;
3876 	struct packet_sock *po = pkt_sk(sk);
3877 	void *data = &val;
3878 	union tpacket_stats_u st;
3879 	struct tpacket_rollover_stats rstats;
3880 
3881 	if (level != SOL_PACKET)
3882 		return -ENOPROTOOPT;
3883 
3884 	if (get_user(len, optlen))
3885 		return -EFAULT;
3886 
3887 	if (len < 0)
3888 		return -EINVAL;
3889 
3890 	switch (optname) {
3891 	case PACKET_STATISTICS:
3892 		spin_lock_bh(&sk->sk_receive_queue.lock);
3893 		memcpy(&st, &po->stats, sizeof(st));
3894 		memset(&po->stats, 0, sizeof(po->stats));
3895 		spin_unlock_bh(&sk->sk_receive_queue.lock);
3896 
3897 		if (po->tp_version == TPACKET_V3) {
3898 			lv = sizeof(struct tpacket_stats_v3);
3899 			st.stats3.tp_packets += st.stats3.tp_drops;
3900 			data = &st.stats3;
3901 		} else {
3902 			lv = sizeof(struct tpacket_stats);
3903 			st.stats1.tp_packets += st.stats1.tp_drops;
3904 			data = &st.stats1;
3905 		}
3906 
3907 		break;
3908 	case PACKET_AUXDATA:
3909 		val = po->auxdata;
3910 		break;
3911 	case PACKET_ORIGDEV:
3912 		val = po->origdev;
3913 		break;
3914 	case PACKET_VNET_HDR:
3915 		val = po->has_vnet_hdr;
3916 		break;
3917 	case PACKET_VERSION:
3918 		val = po->tp_version;
3919 		break;
3920 	case PACKET_HDRLEN:
3921 		if (len > sizeof(int))
3922 			len = sizeof(int);
3923 		if (len < sizeof(int))
3924 			return -EINVAL;
3925 		if (copy_from_user(&val, optval, len))
3926 			return -EFAULT;
3927 		switch (val) {
3928 		case TPACKET_V1:
3929 			val = sizeof(struct tpacket_hdr);
3930 			break;
3931 		case TPACKET_V2:
3932 			val = sizeof(struct tpacket2_hdr);
3933 			break;
3934 		case TPACKET_V3:
3935 			val = sizeof(struct tpacket3_hdr);
3936 			break;
3937 		default:
3938 			return -EINVAL;
3939 		}
3940 		break;
3941 	case PACKET_RESERVE:
3942 		val = po->tp_reserve;
3943 		break;
3944 	case PACKET_LOSS:
3945 		val = po->tp_loss;
3946 		break;
3947 	case PACKET_TIMESTAMP:
3948 		val = po->tp_tstamp;
3949 		break;
3950 	case PACKET_FANOUT:
3951 		val = (po->fanout ?
3952 		       ((u32)po->fanout->id |
3953 			((u32)po->fanout->type << 16) |
3954 			((u32)po->fanout->flags << 24)) :
3955 		       0);
3956 		break;
3957 	case PACKET_ROLLOVER_STATS:
3958 		if (!po->rollover)
3959 			return -EINVAL;
3960 		rstats.tp_all = atomic_long_read(&po->rollover->num);
3961 		rstats.tp_huge = atomic_long_read(&po->rollover->num_huge);
3962 		rstats.tp_failed = atomic_long_read(&po->rollover->num_failed);
3963 		data = &rstats;
3964 		lv = sizeof(rstats);
3965 		break;
3966 	case PACKET_TX_HAS_OFF:
3967 		val = po->tp_tx_has_off;
3968 		break;
3969 	case PACKET_QDISC_BYPASS:
3970 		val = packet_use_direct_xmit(po);
3971 		break;
3972 	default:
3973 		return -ENOPROTOOPT;
3974 	}
3975 
3976 	if (len > lv)
3977 		len = lv;
3978 	if (put_user(len, optlen))
3979 		return -EFAULT;
3980 	if (copy_to_user(optval, data, len))
3981 		return -EFAULT;
3982 	return 0;
3983 }
3984 
3985 
3986 #ifdef CONFIG_COMPAT
3987 static int compat_packet_setsockopt(struct socket *sock, int level, int optname,
3988 				    char __user *optval, unsigned int optlen)
3989 {
3990 	struct packet_sock *po = pkt_sk(sock->sk);
3991 
3992 	if (level != SOL_PACKET)
3993 		return -ENOPROTOOPT;
3994 
3995 	if (optname == PACKET_FANOUT_DATA &&
3996 	    po->fanout && po->fanout->type == PACKET_FANOUT_CBPF) {
3997 		optval = (char __user *)get_compat_bpf_fprog(optval);
3998 		if (!optval)
3999 			return -EFAULT;
4000 		optlen = sizeof(struct sock_fprog);
4001 	}
4002 
4003 	return packet_setsockopt(sock, level, optname, optval, optlen);
4004 }
4005 #endif
4006 
4007 static int packet_notifier(struct notifier_block *this,
4008 			   unsigned long msg, void *ptr)
4009 {
4010 	struct sock *sk;
4011 	struct net_device *dev = netdev_notifier_info_to_dev(ptr);
4012 	struct net *net = dev_net(dev);
4013 
4014 	rcu_read_lock();
4015 	sk_for_each_rcu(sk, &net->packet.sklist) {
4016 		struct packet_sock *po = pkt_sk(sk);
4017 
4018 		switch (msg) {
4019 		case NETDEV_UNREGISTER:
4020 			if (po->mclist)
4021 				packet_dev_mclist_delete(dev, &po->mclist);
4022 			/* fallthrough */
4023 
4024 		case NETDEV_DOWN:
4025 			if (dev->ifindex == po->ifindex) {
4026 				spin_lock(&po->bind_lock);
4027 				if (po->running) {
4028 					__unregister_prot_hook(sk, false);
4029 					sk->sk_err = ENETDOWN;
4030 					if (!sock_flag(sk, SOCK_DEAD))
4031 						sk->sk_error_report(sk);
4032 				}
4033 				if (msg == NETDEV_UNREGISTER) {
4034 					packet_cached_dev_reset(po);
4035 					po->ifindex = -1;
4036 					if (po->prot_hook.dev)
4037 						dev_put(po->prot_hook.dev);
4038 					po->prot_hook.dev = NULL;
4039 				}
4040 				spin_unlock(&po->bind_lock);
4041 			}
4042 			break;
4043 		case NETDEV_UP:
4044 			if (dev->ifindex == po->ifindex) {
4045 				spin_lock(&po->bind_lock);
4046 				if (po->num)
4047 					register_prot_hook(sk);
4048 				spin_unlock(&po->bind_lock);
4049 			}
4050 			break;
4051 		}
4052 	}
4053 	rcu_read_unlock();
4054 	return NOTIFY_DONE;
4055 }
4056 
4057 
4058 static int packet_ioctl(struct socket *sock, unsigned int cmd,
4059 			unsigned long arg)
4060 {
4061 	struct sock *sk = sock->sk;
4062 
4063 	switch (cmd) {
4064 	case SIOCOUTQ:
4065 	{
4066 		int amount = sk_wmem_alloc_get(sk);
4067 
4068 		return put_user(amount, (int __user *)arg);
4069 	}
4070 	case SIOCINQ:
4071 	{
4072 		struct sk_buff *skb;
4073 		int amount = 0;
4074 
4075 		spin_lock_bh(&sk->sk_receive_queue.lock);
4076 		skb = skb_peek(&sk->sk_receive_queue);
4077 		if (skb)
4078 			amount = skb->len;
4079 		spin_unlock_bh(&sk->sk_receive_queue.lock);
4080 		return put_user(amount, (int __user *)arg);
4081 	}
4082 	case SIOCGSTAMP:
4083 		return sock_get_timestamp(sk, (struct timeval __user *)arg);
4084 	case SIOCGSTAMPNS:
4085 		return sock_get_timestampns(sk, (struct timespec __user *)arg);
4086 
4087 #ifdef CONFIG_INET
4088 	case SIOCADDRT:
4089 	case SIOCDELRT:
4090 	case SIOCDARP:
4091 	case SIOCGARP:
4092 	case SIOCSARP:
4093 	case SIOCGIFADDR:
4094 	case SIOCSIFADDR:
4095 	case SIOCGIFBRDADDR:
4096 	case SIOCSIFBRDADDR:
4097 	case SIOCGIFNETMASK:
4098 	case SIOCSIFNETMASK:
4099 	case SIOCGIFDSTADDR:
4100 	case SIOCSIFDSTADDR:
4101 	case SIOCSIFFLAGS:
4102 		return inet_dgram_ops.ioctl(sock, cmd, arg);
4103 #endif
4104 
4105 	default:
4106 		return -ENOIOCTLCMD;
4107 	}
4108 	return 0;
4109 }
4110 
4111 static __poll_t packet_poll(struct file *file, struct socket *sock,
4112 				poll_table *wait)
4113 {
4114 	struct sock *sk = sock->sk;
4115 	struct packet_sock *po = pkt_sk(sk);
4116 	__poll_t mask = datagram_poll(file, sock, wait);
4117 
4118 	spin_lock_bh(&sk->sk_receive_queue.lock);
4119 	if (po->rx_ring.pg_vec) {
4120 		if (!packet_previous_rx_frame(po, &po->rx_ring,
4121 			TP_STATUS_KERNEL))
4122 			mask |= EPOLLIN | EPOLLRDNORM;
4123 	}
4124 	if (po->pressure && __packet_rcv_has_room(po, NULL) == ROOM_NORMAL)
4125 		po->pressure = 0;
4126 	spin_unlock_bh(&sk->sk_receive_queue.lock);
4127 	spin_lock_bh(&sk->sk_write_queue.lock);
4128 	if (po->tx_ring.pg_vec) {
4129 		if (packet_current_frame(po, &po->tx_ring, TP_STATUS_AVAILABLE))
4130 			mask |= EPOLLOUT | EPOLLWRNORM;
4131 	}
4132 	spin_unlock_bh(&sk->sk_write_queue.lock);
4133 	return mask;
4134 }
4135 
4136 
4137 /* Dirty? Well, I still did not learn better way to account
4138  * for user mmaps.
4139  */
4140 
4141 static void packet_mm_open(struct vm_area_struct *vma)
4142 {
4143 	struct file *file = vma->vm_file;
4144 	struct socket *sock = file->private_data;
4145 	struct sock *sk = sock->sk;
4146 
4147 	if (sk)
4148 		atomic_inc(&pkt_sk(sk)->mapped);
4149 }
4150 
4151 static void packet_mm_close(struct vm_area_struct *vma)
4152 {
4153 	struct file *file = vma->vm_file;
4154 	struct socket *sock = file->private_data;
4155 	struct sock *sk = sock->sk;
4156 
4157 	if (sk)
4158 		atomic_dec(&pkt_sk(sk)->mapped);
4159 }
4160 
4161 static const struct vm_operations_struct packet_mmap_ops = {
4162 	.open	=	packet_mm_open,
4163 	.close	=	packet_mm_close,
4164 };
4165 
4166 static void free_pg_vec(struct pgv *pg_vec, unsigned int order,
4167 			unsigned int len)
4168 {
4169 	int i;
4170 
4171 	for (i = 0; i < len; i++) {
4172 		if (likely(pg_vec[i].buffer)) {
4173 			if (is_vmalloc_addr(pg_vec[i].buffer))
4174 				vfree(pg_vec[i].buffer);
4175 			else
4176 				free_pages((unsigned long)pg_vec[i].buffer,
4177 					   order);
4178 			pg_vec[i].buffer = NULL;
4179 		}
4180 	}
4181 	kfree(pg_vec);
4182 }
4183 
4184 static char *alloc_one_pg_vec_page(unsigned long order)
4185 {
4186 	char *buffer;
4187 	gfp_t gfp_flags = GFP_KERNEL | __GFP_COMP |
4188 			  __GFP_ZERO | __GFP_NOWARN | __GFP_NORETRY;
4189 
4190 	buffer = (char *) __get_free_pages(gfp_flags, order);
4191 	if (buffer)
4192 		return buffer;
4193 
4194 	/* __get_free_pages failed, fall back to vmalloc */
4195 	buffer = vzalloc((1 << order) * PAGE_SIZE);
4196 	if (buffer)
4197 		return buffer;
4198 
4199 	/* vmalloc failed, lets dig into swap here */
4200 	gfp_flags &= ~__GFP_NORETRY;
4201 	buffer = (char *) __get_free_pages(gfp_flags, order);
4202 	if (buffer)
4203 		return buffer;
4204 
4205 	/* complete and utter failure */
4206 	return NULL;
4207 }
4208 
4209 static struct pgv *alloc_pg_vec(struct tpacket_req *req, int order)
4210 {
4211 	unsigned int block_nr = req->tp_block_nr;
4212 	struct pgv *pg_vec;
4213 	int i;
4214 
4215 	pg_vec = kcalloc(block_nr, sizeof(struct pgv), GFP_KERNEL);
4216 	if (unlikely(!pg_vec))
4217 		goto out;
4218 
4219 	for (i = 0; i < block_nr; i++) {
4220 		pg_vec[i].buffer = alloc_one_pg_vec_page(order);
4221 		if (unlikely(!pg_vec[i].buffer))
4222 			goto out_free_pgvec;
4223 	}
4224 
4225 out:
4226 	return pg_vec;
4227 
4228 out_free_pgvec:
4229 	free_pg_vec(pg_vec, order, block_nr);
4230 	pg_vec = NULL;
4231 	goto out;
4232 }
4233 
4234 static int packet_set_ring(struct sock *sk, union tpacket_req_u *req_u,
4235 		int closing, int tx_ring)
4236 {
4237 	struct pgv *pg_vec = NULL;
4238 	struct packet_sock *po = pkt_sk(sk);
4239 	int was_running, order = 0;
4240 	struct packet_ring_buffer *rb;
4241 	struct sk_buff_head *rb_queue;
4242 	__be16 num;
4243 	int err = -EINVAL;
4244 	/* Added to avoid minimal code churn */
4245 	struct tpacket_req *req = &req_u->req;
4246 
4247 	rb = tx_ring ? &po->tx_ring : &po->rx_ring;
4248 	rb_queue = tx_ring ? &sk->sk_write_queue : &sk->sk_receive_queue;
4249 
4250 	err = -EBUSY;
4251 	if (!closing) {
4252 		if (atomic_read(&po->mapped))
4253 			goto out;
4254 		if (packet_read_pending(rb))
4255 			goto out;
4256 	}
4257 
4258 	if (req->tp_block_nr) {
4259 		/* Sanity tests and some calculations */
4260 		err = -EBUSY;
4261 		if (unlikely(rb->pg_vec))
4262 			goto out;
4263 
4264 		switch (po->tp_version) {
4265 		case TPACKET_V1:
4266 			po->tp_hdrlen = TPACKET_HDRLEN;
4267 			break;
4268 		case TPACKET_V2:
4269 			po->tp_hdrlen = TPACKET2_HDRLEN;
4270 			break;
4271 		case TPACKET_V3:
4272 			po->tp_hdrlen = TPACKET3_HDRLEN;
4273 			break;
4274 		}
4275 
4276 		err = -EINVAL;
4277 		if (unlikely((int)req->tp_block_size <= 0))
4278 			goto out;
4279 		if (unlikely(!PAGE_ALIGNED(req->tp_block_size)))
4280 			goto out;
4281 		if (po->tp_version >= TPACKET_V3 &&
4282 		    req->tp_block_size <=
4283 			  BLK_PLUS_PRIV((u64)req_u->req3.tp_sizeof_priv))
4284 			goto out;
4285 		if (unlikely(req->tp_frame_size < po->tp_hdrlen +
4286 					po->tp_reserve))
4287 			goto out;
4288 		if (unlikely(req->tp_frame_size & (TPACKET_ALIGNMENT - 1)))
4289 			goto out;
4290 
4291 		rb->frames_per_block = req->tp_block_size / req->tp_frame_size;
4292 		if (unlikely(rb->frames_per_block == 0))
4293 			goto out;
4294 		if (unlikely(req->tp_block_size > UINT_MAX / req->tp_block_nr))
4295 			goto out;
4296 		if (unlikely((rb->frames_per_block * req->tp_block_nr) !=
4297 					req->tp_frame_nr))
4298 			goto out;
4299 
4300 		err = -ENOMEM;
4301 		order = get_order(req->tp_block_size);
4302 		pg_vec = alloc_pg_vec(req, order);
4303 		if (unlikely(!pg_vec))
4304 			goto out;
4305 		switch (po->tp_version) {
4306 		case TPACKET_V3:
4307 			/* Block transmit is not supported yet */
4308 			if (!tx_ring) {
4309 				init_prb_bdqc(po, rb, pg_vec, req_u);
4310 			} else {
4311 				struct tpacket_req3 *req3 = &req_u->req3;
4312 
4313 				if (req3->tp_retire_blk_tov ||
4314 				    req3->tp_sizeof_priv ||
4315 				    req3->tp_feature_req_word) {
4316 					err = -EINVAL;
4317 					goto out;
4318 				}
4319 			}
4320 			break;
4321 		default:
4322 			break;
4323 		}
4324 	}
4325 	/* Done */
4326 	else {
4327 		err = -EINVAL;
4328 		if (unlikely(req->tp_frame_nr))
4329 			goto out;
4330 	}
4331 
4332 
4333 	/* Detach socket from network */
4334 	spin_lock(&po->bind_lock);
4335 	was_running = po->running;
4336 	num = po->num;
4337 	if (was_running) {
4338 		po->num = 0;
4339 		__unregister_prot_hook(sk, false);
4340 	}
4341 	spin_unlock(&po->bind_lock);
4342 
4343 	synchronize_net();
4344 
4345 	err = -EBUSY;
4346 	mutex_lock(&po->pg_vec_lock);
4347 	if (closing || atomic_read(&po->mapped) == 0) {
4348 		err = 0;
4349 		spin_lock_bh(&rb_queue->lock);
4350 		swap(rb->pg_vec, pg_vec);
4351 		rb->frame_max = (req->tp_frame_nr - 1);
4352 		rb->head = 0;
4353 		rb->frame_size = req->tp_frame_size;
4354 		spin_unlock_bh(&rb_queue->lock);
4355 
4356 		swap(rb->pg_vec_order, order);
4357 		swap(rb->pg_vec_len, req->tp_block_nr);
4358 
4359 		rb->pg_vec_pages = req->tp_block_size/PAGE_SIZE;
4360 		po->prot_hook.func = (po->rx_ring.pg_vec) ?
4361 						tpacket_rcv : packet_rcv;
4362 		skb_queue_purge(rb_queue);
4363 		if (atomic_read(&po->mapped))
4364 			pr_err("packet_mmap: vma is busy: %d\n",
4365 			       atomic_read(&po->mapped));
4366 	}
4367 	mutex_unlock(&po->pg_vec_lock);
4368 
4369 	spin_lock(&po->bind_lock);
4370 	if (was_running) {
4371 		po->num = num;
4372 		register_prot_hook(sk);
4373 	}
4374 	spin_unlock(&po->bind_lock);
4375 	if (pg_vec && (po->tp_version > TPACKET_V2)) {
4376 		/* Because we don't support block-based V3 on tx-ring */
4377 		if (!tx_ring)
4378 			prb_shutdown_retire_blk_timer(po, rb_queue);
4379 	}
4380 
4381 	if (pg_vec)
4382 		free_pg_vec(pg_vec, order, req->tp_block_nr);
4383 out:
4384 	return err;
4385 }
4386 
4387 static int packet_mmap(struct file *file, struct socket *sock,
4388 		struct vm_area_struct *vma)
4389 {
4390 	struct sock *sk = sock->sk;
4391 	struct packet_sock *po = pkt_sk(sk);
4392 	unsigned long size, expected_size;
4393 	struct packet_ring_buffer *rb;
4394 	unsigned long start;
4395 	int err = -EINVAL;
4396 	int i;
4397 
4398 	if (vma->vm_pgoff)
4399 		return -EINVAL;
4400 
4401 	mutex_lock(&po->pg_vec_lock);
4402 
4403 	expected_size = 0;
4404 	for (rb = &po->rx_ring; rb <= &po->tx_ring; rb++) {
4405 		if (rb->pg_vec) {
4406 			expected_size += rb->pg_vec_len
4407 						* rb->pg_vec_pages
4408 						* PAGE_SIZE;
4409 		}
4410 	}
4411 
4412 	if (expected_size == 0)
4413 		goto out;
4414 
4415 	size = vma->vm_end - vma->vm_start;
4416 	if (size != expected_size)
4417 		goto out;
4418 
4419 	start = vma->vm_start;
4420 	for (rb = &po->rx_ring; rb <= &po->tx_ring; rb++) {
4421 		if (rb->pg_vec == NULL)
4422 			continue;
4423 
4424 		for (i = 0; i < rb->pg_vec_len; i++) {
4425 			struct page *page;
4426 			void *kaddr = rb->pg_vec[i].buffer;
4427 			int pg_num;
4428 
4429 			for (pg_num = 0; pg_num < rb->pg_vec_pages; pg_num++) {
4430 				page = pgv_to_page(kaddr);
4431 				err = vm_insert_page(vma, start, page);
4432 				if (unlikely(err))
4433 					goto out;
4434 				start += PAGE_SIZE;
4435 				kaddr += PAGE_SIZE;
4436 			}
4437 		}
4438 	}
4439 
4440 	atomic_inc(&po->mapped);
4441 	vma->vm_ops = &packet_mmap_ops;
4442 	err = 0;
4443 
4444 out:
4445 	mutex_unlock(&po->pg_vec_lock);
4446 	return err;
4447 }
4448 
4449 static const struct proto_ops packet_ops_spkt = {
4450 	.family =	PF_PACKET,
4451 	.owner =	THIS_MODULE,
4452 	.release =	packet_release,
4453 	.bind =		packet_bind_spkt,
4454 	.connect =	sock_no_connect,
4455 	.socketpair =	sock_no_socketpair,
4456 	.accept =	sock_no_accept,
4457 	.getname =	packet_getname_spkt,
4458 	.poll =		datagram_poll,
4459 	.ioctl =	packet_ioctl,
4460 	.listen =	sock_no_listen,
4461 	.shutdown =	sock_no_shutdown,
4462 	.setsockopt =	sock_no_setsockopt,
4463 	.getsockopt =	sock_no_getsockopt,
4464 	.sendmsg =	packet_sendmsg_spkt,
4465 	.recvmsg =	packet_recvmsg,
4466 	.mmap =		sock_no_mmap,
4467 	.sendpage =	sock_no_sendpage,
4468 };
4469 
4470 static const struct proto_ops packet_ops = {
4471 	.family =	PF_PACKET,
4472 	.owner =	THIS_MODULE,
4473 	.release =	packet_release,
4474 	.bind =		packet_bind,
4475 	.connect =	sock_no_connect,
4476 	.socketpair =	sock_no_socketpair,
4477 	.accept =	sock_no_accept,
4478 	.getname =	packet_getname,
4479 	.poll =		packet_poll,
4480 	.ioctl =	packet_ioctl,
4481 	.listen =	sock_no_listen,
4482 	.shutdown =	sock_no_shutdown,
4483 	.setsockopt =	packet_setsockopt,
4484 	.getsockopt =	packet_getsockopt,
4485 #ifdef CONFIG_COMPAT
4486 	.compat_setsockopt = compat_packet_setsockopt,
4487 #endif
4488 	.sendmsg =	packet_sendmsg,
4489 	.recvmsg =	packet_recvmsg,
4490 	.mmap =		packet_mmap,
4491 	.sendpage =	sock_no_sendpage,
4492 };
4493 
4494 static const struct net_proto_family packet_family_ops = {
4495 	.family =	PF_PACKET,
4496 	.create =	packet_create,
4497 	.owner	=	THIS_MODULE,
4498 };
4499 
4500 static struct notifier_block packet_netdev_notifier = {
4501 	.notifier_call =	packet_notifier,
4502 };
4503 
4504 #ifdef CONFIG_PROC_FS
4505 
4506 static void *packet_seq_start(struct seq_file *seq, loff_t *pos)
4507 	__acquires(RCU)
4508 {
4509 	struct net *net = seq_file_net(seq);
4510 
4511 	rcu_read_lock();
4512 	return seq_hlist_start_head_rcu(&net->packet.sklist, *pos);
4513 }
4514 
4515 static void *packet_seq_next(struct seq_file *seq, void *v, loff_t *pos)
4516 {
4517 	struct net *net = seq_file_net(seq);
4518 	return seq_hlist_next_rcu(v, &net->packet.sklist, pos);
4519 }
4520 
4521 static void packet_seq_stop(struct seq_file *seq, void *v)
4522 	__releases(RCU)
4523 {
4524 	rcu_read_unlock();
4525 }
4526 
4527 static int packet_seq_show(struct seq_file *seq, void *v)
4528 {
4529 	if (v == SEQ_START_TOKEN)
4530 		seq_puts(seq, "sk       RefCnt Type Proto  Iface R Rmem   User   Inode\n");
4531 	else {
4532 		struct sock *s = sk_entry(v);
4533 		const struct packet_sock *po = pkt_sk(s);
4534 
4535 		seq_printf(seq,
4536 			   "%pK %-6d %-4d %04x   %-5d %1d %-6u %-6u %-6lu\n",
4537 			   s,
4538 			   refcount_read(&s->sk_refcnt),
4539 			   s->sk_type,
4540 			   ntohs(po->num),
4541 			   po->ifindex,
4542 			   po->running,
4543 			   atomic_read(&s->sk_rmem_alloc),
4544 			   from_kuid_munged(seq_user_ns(seq), sock_i_uid(s)),
4545 			   sock_i_ino(s));
4546 	}
4547 
4548 	return 0;
4549 }
4550 
4551 static const struct seq_operations packet_seq_ops = {
4552 	.start	= packet_seq_start,
4553 	.next	= packet_seq_next,
4554 	.stop	= packet_seq_stop,
4555 	.show	= packet_seq_show,
4556 };
4557 
4558 static int packet_seq_open(struct inode *inode, struct file *file)
4559 {
4560 	return seq_open_net(inode, file, &packet_seq_ops,
4561 			    sizeof(struct seq_net_private));
4562 }
4563 
4564 static const struct file_operations packet_seq_fops = {
4565 	.open		= packet_seq_open,
4566 	.read		= seq_read,
4567 	.llseek		= seq_lseek,
4568 	.release	= seq_release_net,
4569 };
4570 
4571 #endif
4572 
4573 static int __net_init packet_net_init(struct net *net)
4574 {
4575 	mutex_init(&net->packet.sklist_lock);
4576 	INIT_HLIST_HEAD(&net->packet.sklist);
4577 
4578 	if (!proc_create("packet", 0, net->proc_net, &packet_seq_fops))
4579 		return -ENOMEM;
4580 
4581 	return 0;
4582 }
4583 
4584 static void __net_exit packet_net_exit(struct net *net)
4585 {
4586 	remove_proc_entry("packet", net->proc_net);
4587 	WARN_ON_ONCE(!hlist_empty(&net->packet.sklist));
4588 }
4589 
4590 static struct pernet_operations packet_net_ops = {
4591 	.init = packet_net_init,
4592 	.exit = packet_net_exit,
4593 };
4594 
4595 
4596 static void __exit packet_exit(void)
4597 {
4598 	unregister_netdevice_notifier(&packet_netdev_notifier);
4599 	unregister_pernet_subsys(&packet_net_ops);
4600 	sock_unregister(PF_PACKET);
4601 	proto_unregister(&packet_proto);
4602 }
4603 
4604 static int __init packet_init(void)
4605 {
4606 	int rc = proto_register(&packet_proto, 0);
4607 
4608 	if (rc != 0)
4609 		goto out;
4610 
4611 	sock_register(&packet_family_ops);
4612 	register_pernet_subsys(&packet_net_ops);
4613 	register_netdevice_notifier(&packet_netdev_notifier);
4614 out:
4615 	return rc;
4616 }
4617 
4618 module_init(packet_init);
4619 module_exit(packet_exit);
4620 MODULE_LICENSE("GPL");
4621 MODULE_ALIAS_NETPROTO(PF_PACKET);
4622