xref: /openbmc/linux/net/packet/af_packet.c (revision afc98d90)
1 /*
2  * INET		An implementation of the TCP/IP protocol suite for the LINUX
3  *		operating system.  INET is implemented using the  BSD Socket
4  *		interface as the means of communication with the user level.
5  *
6  *		PACKET - implements raw packet sockets.
7  *
8  * Authors:	Ross Biro
9  *		Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
10  *		Alan Cox, <gw4pts@gw4pts.ampr.org>
11  *
12  * Fixes:
13  *		Alan Cox	:	verify_area() now used correctly
14  *		Alan Cox	:	new skbuff lists, look ma no backlogs!
15  *		Alan Cox	:	tidied skbuff lists.
16  *		Alan Cox	:	Now uses generic datagram routines I
17  *					added. Also fixed the peek/read crash
18  *					from all old Linux datagram code.
19  *		Alan Cox	:	Uses the improved datagram code.
20  *		Alan Cox	:	Added NULL's for socket options.
21  *		Alan Cox	:	Re-commented the code.
22  *		Alan Cox	:	Use new kernel side addressing
23  *		Rob Janssen	:	Correct MTU usage.
24  *		Dave Platt	:	Counter leaks caused by incorrect
25  *					interrupt locking and some slightly
26  *					dubious gcc output. Can you read
27  *					compiler: it said _VOLATILE_
28  *	Richard Kooijman	:	Timestamp fixes.
29  *		Alan Cox	:	New buffers. Use sk->mac.raw.
30  *		Alan Cox	:	sendmsg/recvmsg support.
31  *		Alan Cox	:	Protocol setting support
32  *	Alexey Kuznetsov	:	Untied from IPv4 stack.
33  *	Cyrus Durgin		:	Fixed kerneld for kmod.
34  *	Michal Ostrowski        :       Module initialization cleanup.
35  *         Ulises Alonso        :       Frame number limit removal and
36  *                                      packet_set_ring memory leak.
37  *		Eric Biederman	:	Allow for > 8 byte hardware addresses.
38  *					The convention is that longer addresses
39  *					will simply extend the hardware address
40  *					byte arrays at the end of sockaddr_ll
41  *					and packet_mreq.
42  *		Johann Baudy	:	Added TX RING.
43  *		Chetan Loke	:	Implemented TPACKET_V3 block abstraction
44  *					layer.
45  *					Copyright (C) 2011, <lokec@ccs.neu.edu>
46  *
47  *
48  *		This program is free software; you can redistribute it and/or
49  *		modify it under the terms of the GNU General Public License
50  *		as published by the Free Software Foundation; either version
51  *		2 of the License, or (at your option) any later version.
52  *
53  */
54 
55 #include <linux/types.h>
56 #include <linux/mm.h>
57 #include <linux/capability.h>
58 #include <linux/fcntl.h>
59 #include <linux/socket.h>
60 #include <linux/in.h>
61 #include <linux/inet.h>
62 #include <linux/netdevice.h>
63 #include <linux/if_packet.h>
64 #include <linux/wireless.h>
65 #include <linux/kernel.h>
66 #include <linux/kmod.h>
67 #include <linux/slab.h>
68 #include <linux/vmalloc.h>
69 #include <net/net_namespace.h>
70 #include <net/ip.h>
71 #include <net/protocol.h>
72 #include <linux/skbuff.h>
73 #include <net/sock.h>
74 #include <linux/errno.h>
75 #include <linux/timer.h>
76 #include <asm/uaccess.h>
77 #include <asm/ioctls.h>
78 #include <asm/page.h>
79 #include <asm/cacheflush.h>
80 #include <asm/io.h>
81 #include <linux/proc_fs.h>
82 #include <linux/seq_file.h>
83 #include <linux/poll.h>
84 #include <linux/module.h>
85 #include <linux/init.h>
86 #include <linux/mutex.h>
87 #include <linux/if_vlan.h>
88 #include <linux/virtio_net.h>
89 #include <linux/errqueue.h>
90 #include <linux/net_tstamp.h>
91 #include <linux/percpu.h>
92 #ifdef CONFIG_INET
93 #include <net/inet_common.h>
94 #endif
95 
96 #include "internal.h"
97 
98 /*
99    Assumptions:
100    - if device has no dev->hard_header routine, it adds and removes ll header
101      inside itself. In this case ll header is invisible outside of device,
102      but higher levels still should reserve dev->hard_header_len.
103      Some devices are enough clever to reallocate skb, when header
104      will not fit to reserved space (tunnel), another ones are silly
105      (PPP).
106    - packet socket receives packets with pulled ll header,
107      so that SOCK_RAW should push it back.
108 
109 On receive:
110 -----------
111 
112 Incoming, dev->hard_header!=NULL
113    mac_header -> ll header
114    data       -> data
115 
116 Outgoing, dev->hard_header!=NULL
117    mac_header -> ll header
118    data       -> ll header
119 
120 Incoming, dev->hard_header==NULL
121    mac_header -> UNKNOWN position. It is very likely, that it points to ll
122 		 header.  PPP makes it, that is wrong, because introduce
123 		 assymetry between rx and tx paths.
124    data       -> data
125 
126 Outgoing, dev->hard_header==NULL
127    mac_header -> data. ll header is still not built!
128    data       -> data
129 
130 Resume
131   If dev->hard_header==NULL we are unlikely to restore sensible ll header.
132 
133 
134 On transmit:
135 ------------
136 
137 dev->hard_header != NULL
138    mac_header -> ll header
139    data       -> ll header
140 
141 dev->hard_header == NULL (ll header is added by device, we cannot control it)
142    mac_header -> data
143    data       -> data
144 
145    We should set nh.raw on output to correct posistion,
146    packet classifier depends on it.
147  */
148 
149 /* Private packet socket structures. */
150 
151 /* identical to struct packet_mreq except it has
152  * a longer address field.
153  */
154 struct packet_mreq_max {
155 	int		mr_ifindex;
156 	unsigned short	mr_type;
157 	unsigned short	mr_alen;
158 	unsigned char	mr_address[MAX_ADDR_LEN];
159 };
160 
161 union tpacket_uhdr {
162 	struct tpacket_hdr  *h1;
163 	struct tpacket2_hdr *h2;
164 	struct tpacket3_hdr *h3;
165 	void *raw;
166 };
167 
168 static int packet_set_ring(struct sock *sk, union tpacket_req_u *req_u,
169 		int closing, int tx_ring);
170 
171 #define V3_ALIGNMENT	(8)
172 
173 #define BLK_HDR_LEN	(ALIGN(sizeof(struct tpacket_block_desc), V3_ALIGNMENT))
174 
175 #define BLK_PLUS_PRIV(sz_of_priv) \
176 	(BLK_HDR_LEN + ALIGN((sz_of_priv), V3_ALIGNMENT))
177 
178 #define PGV_FROM_VMALLOC 1
179 
180 #define BLOCK_STATUS(x)	((x)->hdr.bh1.block_status)
181 #define BLOCK_NUM_PKTS(x)	((x)->hdr.bh1.num_pkts)
182 #define BLOCK_O2FP(x)		((x)->hdr.bh1.offset_to_first_pkt)
183 #define BLOCK_LEN(x)		((x)->hdr.bh1.blk_len)
184 #define BLOCK_SNUM(x)		((x)->hdr.bh1.seq_num)
185 #define BLOCK_O2PRIV(x)	((x)->offset_to_priv)
186 #define BLOCK_PRIV(x)		((void *)((char *)(x) + BLOCK_O2PRIV(x)))
187 
188 struct packet_sock;
189 static int tpacket_snd(struct packet_sock *po, struct msghdr *msg);
190 static int tpacket_rcv(struct sk_buff *skb, struct net_device *dev,
191 		       struct packet_type *pt, struct net_device *orig_dev);
192 
193 static void *packet_previous_frame(struct packet_sock *po,
194 		struct packet_ring_buffer *rb,
195 		int status);
196 static void packet_increment_head(struct packet_ring_buffer *buff);
197 static int prb_curr_blk_in_use(struct tpacket_kbdq_core *,
198 			struct tpacket_block_desc *);
199 static void *prb_dispatch_next_block(struct tpacket_kbdq_core *,
200 			struct packet_sock *);
201 static void prb_retire_current_block(struct tpacket_kbdq_core *,
202 		struct packet_sock *, unsigned int status);
203 static int prb_queue_frozen(struct tpacket_kbdq_core *);
204 static void prb_open_block(struct tpacket_kbdq_core *,
205 		struct tpacket_block_desc *);
206 static void prb_retire_rx_blk_timer_expired(unsigned long);
207 static void _prb_refresh_rx_retire_blk_timer(struct tpacket_kbdq_core *);
208 static void prb_init_blk_timer(struct packet_sock *,
209 		struct tpacket_kbdq_core *,
210 		void (*func) (unsigned long));
211 static void prb_fill_rxhash(struct tpacket_kbdq_core *, struct tpacket3_hdr *);
212 static void prb_clear_rxhash(struct tpacket_kbdq_core *,
213 		struct tpacket3_hdr *);
214 static void prb_fill_vlan_info(struct tpacket_kbdq_core *,
215 		struct tpacket3_hdr *);
216 static void packet_flush_mclist(struct sock *sk);
217 
218 struct packet_skb_cb {
219 	unsigned int origlen;
220 	union {
221 		struct sockaddr_pkt pkt;
222 		struct sockaddr_ll ll;
223 	} sa;
224 };
225 
226 #define PACKET_SKB_CB(__skb)	((struct packet_skb_cb *)((__skb)->cb))
227 
228 #define GET_PBDQC_FROM_RB(x)	((struct tpacket_kbdq_core *)(&(x)->prb_bdqc))
229 #define GET_PBLOCK_DESC(x, bid)	\
230 	((struct tpacket_block_desc *)((x)->pkbdq[(bid)].buffer))
231 #define GET_CURR_PBLOCK_DESC_FROM_CORE(x)	\
232 	((struct tpacket_block_desc *)((x)->pkbdq[(x)->kactive_blk_num].buffer))
233 #define GET_NEXT_PRB_BLK_NUM(x) \
234 	(((x)->kactive_blk_num < ((x)->knum_blocks-1)) ? \
235 	((x)->kactive_blk_num+1) : 0)
236 
237 static void __fanout_unlink(struct sock *sk, struct packet_sock *po);
238 static void __fanout_link(struct sock *sk, struct packet_sock *po);
239 
240 static int packet_direct_xmit(struct sk_buff *skb)
241 {
242 	struct net_device *dev = skb->dev;
243 	const struct net_device_ops *ops = dev->netdev_ops;
244 	netdev_features_t features;
245 	struct netdev_queue *txq;
246 	u16 queue_map;
247 	int ret;
248 
249 	if (unlikely(!netif_running(dev) ||
250 		     !netif_carrier_ok(dev))) {
251 		kfree_skb(skb);
252 		return NET_XMIT_DROP;
253 	}
254 
255 	features = netif_skb_features(skb);
256 	if (skb_needs_linearize(skb, features) &&
257 	    __skb_linearize(skb)) {
258 		kfree_skb(skb);
259 		return NET_XMIT_DROP;
260 	}
261 
262 	queue_map = skb_get_queue_mapping(skb);
263 	txq = netdev_get_tx_queue(dev, queue_map);
264 
265 	__netif_tx_lock_bh(txq);
266 	if (unlikely(netif_xmit_frozen_or_stopped(txq))) {
267 		ret = NETDEV_TX_BUSY;
268 		kfree_skb(skb);
269 		goto out;
270 	}
271 
272 	ret = ops->ndo_start_xmit(skb, dev);
273 	if (likely(dev_xmit_complete(ret)))
274 		txq_trans_update(txq);
275 	else
276 		kfree_skb(skb);
277 out:
278 	__netif_tx_unlock_bh(txq);
279 	return ret;
280 }
281 
282 static struct net_device *packet_cached_dev_get(struct packet_sock *po)
283 {
284 	struct net_device *dev;
285 
286 	rcu_read_lock();
287 	dev = rcu_dereference(po->cached_dev);
288 	if (likely(dev))
289 		dev_hold(dev);
290 	rcu_read_unlock();
291 
292 	return dev;
293 }
294 
295 static void packet_cached_dev_assign(struct packet_sock *po,
296 				     struct net_device *dev)
297 {
298 	rcu_assign_pointer(po->cached_dev, dev);
299 }
300 
301 static void packet_cached_dev_reset(struct packet_sock *po)
302 {
303 	RCU_INIT_POINTER(po->cached_dev, NULL);
304 }
305 
306 static bool packet_use_direct_xmit(const struct packet_sock *po)
307 {
308 	return po->xmit == packet_direct_xmit;
309 }
310 
311 static u16 packet_pick_tx_queue(struct net_device *dev)
312 {
313 	return (u16) raw_smp_processor_id() % dev->real_num_tx_queues;
314 }
315 
316 /* register_prot_hook must be invoked with the po->bind_lock held,
317  * or from a context in which asynchronous accesses to the packet
318  * socket is not possible (packet_create()).
319  */
320 static void register_prot_hook(struct sock *sk)
321 {
322 	struct packet_sock *po = pkt_sk(sk);
323 
324 	if (!po->running) {
325 		if (po->fanout)
326 			__fanout_link(sk, po);
327 		else
328 			dev_add_pack(&po->prot_hook);
329 
330 		sock_hold(sk);
331 		po->running = 1;
332 	}
333 }
334 
335 /* {,__}unregister_prot_hook() must be invoked with the po->bind_lock
336  * held.   If the sync parameter is true, we will temporarily drop
337  * the po->bind_lock and do a synchronize_net to make sure no
338  * asynchronous packet processing paths still refer to the elements
339  * of po->prot_hook.  If the sync parameter is false, it is the
340  * callers responsibility to take care of this.
341  */
342 static void __unregister_prot_hook(struct sock *sk, bool sync)
343 {
344 	struct packet_sock *po = pkt_sk(sk);
345 
346 	po->running = 0;
347 
348 	if (po->fanout)
349 		__fanout_unlink(sk, po);
350 	else
351 		__dev_remove_pack(&po->prot_hook);
352 
353 	__sock_put(sk);
354 
355 	if (sync) {
356 		spin_unlock(&po->bind_lock);
357 		synchronize_net();
358 		spin_lock(&po->bind_lock);
359 	}
360 }
361 
362 static void unregister_prot_hook(struct sock *sk, bool sync)
363 {
364 	struct packet_sock *po = pkt_sk(sk);
365 
366 	if (po->running)
367 		__unregister_prot_hook(sk, sync);
368 }
369 
370 static inline __pure struct page *pgv_to_page(void *addr)
371 {
372 	if (is_vmalloc_addr(addr))
373 		return vmalloc_to_page(addr);
374 	return virt_to_page(addr);
375 }
376 
377 static void __packet_set_status(struct packet_sock *po, void *frame, int status)
378 {
379 	union tpacket_uhdr h;
380 
381 	h.raw = frame;
382 	switch (po->tp_version) {
383 	case TPACKET_V1:
384 		h.h1->tp_status = status;
385 		flush_dcache_page(pgv_to_page(&h.h1->tp_status));
386 		break;
387 	case TPACKET_V2:
388 		h.h2->tp_status = status;
389 		flush_dcache_page(pgv_to_page(&h.h2->tp_status));
390 		break;
391 	case TPACKET_V3:
392 	default:
393 		WARN(1, "TPACKET version not supported.\n");
394 		BUG();
395 	}
396 
397 	smp_wmb();
398 }
399 
400 static int __packet_get_status(struct packet_sock *po, void *frame)
401 {
402 	union tpacket_uhdr h;
403 
404 	smp_rmb();
405 
406 	h.raw = frame;
407 	switch (po->tp_version) {
408 	case TPACKET_V1:
409 		flush_dcache_page(pgv_to_page(&h.h1->tp_status));
410 		return h.h1->tp_status;
411 	case TPACKET_V2:
412 		flush_dcache_page(pgv_to_page(&h.h2->tp_status));
413 		return h.h2->tp_status;
414 	case TPACKET_V3:
415 	default:
416 		WARN(1, "TPACKET version not supported.\n");
417 		BUG();
418 		return 0;
419 	}
420 }
421 
422 static __u32 tpacket_get_timestamp(struct sk_buff *skb, struct timespec *ts,
423 				   unsigned int flags)
424 {
425 	struct skb_shared_hwtstamps *shhwtstamps = skb_hwtstamps(skb);
426 
427 	if (shhwtstamps) {
428 		if ((flags & SOF_TIMESTAMPING_SYS_HARDWARE) &&
429 		    ktime_to_timespec_cond(shhwtstamps->syststamp, ts))
430 			return TP_STATUS_TS_SYS_HARDWARE;
431 		if ((flags & SOF_TIMESTAMPING_RAW_HARDWARE) &&
432 		    ktime_to_timespec_cond(shhwtstamps->hwtstamp, ts))
433 			return TP_STATUS_TS_RAW_HARDWARE;
434 	}
435 
436 	if (ktime_to_timespec_cond(skb->tstamp, ts))
437 		return TP_STATUS_TS_SOFTWARE;
438 
439 	return 0;
440 }
441 
442 static __u32 __packet_set_timestamp(struct packet_sock *po, void *frame,
443 				    struct sk_buff *skb)
444 {
445 	union tpacket_uhdr h;
446 	struct timespec ts;
447 	__u32 ts_status;
448 
449 	if (!(ts_status = tpacket_get_timestamp(skb, &ts, po->tp_tstamp)))
450 		return 0;
451 
452 	h.raw = frame;
453 	switch (po->tp_version) {
454 	case TPACKET_V1:
455 		h.h1->tp_sec = ts.tv_sec;
456 		h.h1->tp_usec = ts.tv_nsec / NSEC_PER_USEC;
457 		break;
458 	case TPACKET_V2:
459 		h.h2->tp_sec = ts.tv_sec;
460 		h.h2->tp_nsec = ts.tv_nsec;
461 		break;
462 	case TPACKET_V3:
463 	default:
464 		WARN(1, "TPACKET version not supported.\n");
465 		BUG();
466 	}
467 
468 	/* one flush is safe, as both fields always lie on the same cacheline */
469 	flush_dcache_page(pgv_to_page(&h.h1->tp_sec));
470 	smp_wmb();
471 
472 	return ts_status;
473 }
474 
475 static void *packet_lookup_frame(struct packet_sock *po,
476 		struct packet_ring_buffer *rb,
477 		unsigned int position,
478 		int status)
479 {
480 	unsigned int pg_vec_pos, frame_offset;
481 	union tpacket_uhdr h;
482 
483 	pg_vec_pos = position / rb->frames_per_block;
484 	frame_offset = position % rb->frames_per_block;
485 
486 	h.raw = rb->pg_vec[pg_vec_pos].buffer +
487 		(frame_offset * rb->frame_size);
488 
489 	if (status != __packet_get_status(po, h.raw))
490 		return NULL;
491 
492 	return h.raw;
493 }
494 
495 static void *packet_current_frame(struct packet_sock *po,
496 		struct packet_ring_buffer *rb,
497 		int status)
498 {
499 	return packet_lookup_frame(po, rb, rb->head, status);
500 }
501 
502 static void prb_del_retire_blk_timer(struct tpacket_kbdq_core *pkc)
503 {
504 	del_timer_sync(&pkc->retire_blk_timer);
505 }
506 
507 static void prb_shutdown_retire_blk_timer(struct packet_sock *po,
508 		int tx_ring,
509 		struct sk_buff_head *rb_queue)
510 {
511 	struct tpacket_kbdq_core *pkc;
512 
513 	pkc = tx_ring ? GET_PBDQC_FROM_RB(&po->tx_ring) :
514 			GET_PBDQC_FROM_RB(&po->rx_ring);
515 
516 	spin_lock_bh(&rb_queue->lock);
517 	pkc->delete_blk_timer = 1;
518 	spin_unlock_bh(&rb_queue->lock);
519 
520 	prb_del_retire_blk_timer(pkc);
521 }
522 
523 static void prb_init_blk_timer(struct packet_sock *po,
524 		struct tpacket_kbdq_core *pkc,
525 		void (*func) (unsigned long))
526 {
527 	init_timer(&pkc->retire_blk_timer);
528 	pkc->retire_blk_timer.data = (long)po;
529 	pkc->retire_blk_timer.function = func;
530 	pkc->retire_blk_timer.expires = jiffies;
531 }
532 
533 static void prb_setup_retire_blk_timer(struct packet_sock *po, int tx_ring)
534 {
535 	struct tpacket_kbdq_core *pkc;
536 
537 	if (tx_ring)
538 		BUG();
539 
540 	pkc = tx_ring ? GET_PBDQC_FROM_RB(&po->tx_ring) :
541 			GET_PBDQC_FROM_RB(&po->rx_ring);
542 	prb_init_blk_timer(po, pkc, prb_retire_rx_blk_timer_expired);
543 }
544 
545 static int prb_calc_retire_blk_tmo(struct packet_sock *po,
546 				int blk_size_in_bytes)
547 {
548 	struct net_device *dev;
549 	unsigned int mbits = 0, msec = 0, div = 0, tmo = 0;
550 	struct ethtool_cmd ecmd;
551 	int err;
552 	u32 speed;
553 
554 	rtnl_lock();
555 	dev = __dev_get_by_index(sock_net(&po->sk), po->ifindex);
556 	if (unlikely(!dev)) {
557 		rtnl_unlock();
558 		return DEFAULT_PRB_RETIRE_TOV;
559 	}
560 	err = __ethtool_get_settings(dev, &ecmd);
561 	speed = ethtool_cmd_speed(&ecmd);
562 	rtnl_unlock();
563 	if (!err) {
564 		/*
565 		 * If the link speed is so slow you don't really
566 		 * need to worry about perf anyways
567 		 */
568 		if (speed < SPEED_1000 || speed == SPEED_UNKNOWN) {
569 			return DEFAULT_PRB_RETIRE_TOV;
570 		} else {
571 			msec = 1;
572 			div = speed / 1000;
573 		}
574 	}
575 
576 	mbits = (blk_size_in_bytes * 8) / (1024 * 1024);
577 
578 	if (div)
579 		mbits /= div;
580 
581 	tmo = mbits * msec;
582 
583 	if (div)
584 		return tmo+1;
585 	return tmo;
586 }
587 
588 static void prb_init_ft_ops(struct tpacket_kbdq_core *p1,
589 			union tpacket_req_u *req_u)
590 {
591 	p1->feature_req_word = req_u->req3.tp_feature_req_word;
592 }
593 
594 static void init_prb_bdqc(struct packet_sock *po,
595 			struct packet_ring_buffer *rb,
596 			struct pgv *pg_vec,
597 			union tpacket_req_u *req_u, int tx_ring)
598 {
599 	struct tpacket_kbdq_core *p1 = GET_PBDQC_FROM_RB(rb);
600 	struct tpacket_block_desc *pbd;
601 
602 	memset(p1, 0x0, sizeof(*p1));
603 
604 	p1->knxt_seq_num = 1;
605 	p1->pkbdq = pg_vec;
606 	pbd = (struct tpacket_block_desc *)pg_vec[0].buffer;
607 	p1->pkblk_start	= pg_vec[0].buffer;
608 	p1->kblk_size = req_u->req3.tp_block_size;
609 	p1->knum_blocks	= req_u->req3.tp_block_nr;
610 	p1->hdrlen = po->tp_hdrlen;
611 	p1->version = po->tp_version;
612 	p1->last_kactive_blk_num = 0;
613 	po->stats.stats3.tp_freeze_q_cnt = 0;
614 	if (req_u->req3.tp_retire_blk_tov)
615 		p1->retire_blk_tov = req_u->req3.tp_retire_blk_tov;
616 	else
617 		p1->retire_blk_tov = prb_calc_retire_blk_tmo(po,
618 						req_u->req3.tp_block_size);
619 	p1->tov_in_jiffies = msecs_to_jiffies(p1->retire_blk_tov);
620 	p1->blk_sizeof_priv = req_u->req3.tp_sizeof_priv;
621 
622 	prb_init_ft_ops(p1, req_u);
623 	prb_setup_retire_blk_timer(po, tx_ring);
624 	prb_open_block(p1, pbd);
625 }
626 
627 /*  Do NOT update the last_blk_num first.
628  *  Assumes sk_buff_head lock is held.
629  */
630 static void _prb_refresh_rx_retire_blk_timer(struct tpacket_kbdq_core *pkc)
631 {
632 	mod_timer(&pkc->retire_blk_timer,
633 			jiffies + pkc->tov_in_jiffies);
634 	pkc->last_kactive_blk_num = pkc->kactive_blk_num;
635 }
636 
637 /*
638  * Timer logic:
639  * 1) We refresh the timer only when we open a block.
640  *    By doing this we don't waste cycles refreshing the timer
641  *	  on packet-by-packet basis.
642  *
643  * With a 1MB block-size, on a 1Gbps line, it will take
644  * i) ~8 ms to fill a block + ii) memcpy etc.
645  * In this cut we are not accounting for the memcpy time.
646  *
647  * So, if the user sets the 'tmo' to 10ms then the timer
648  * will never fire while the block is still getting filled
649  * (which is what we want). However, the user could choose
650  * to close a block early and that's fine.
651  *
652  * But when the timer does fire, we check whether or not to refresh it.
653  * Since the tmo granularity is in msecs, it is not too expensive
654  * to refresh the timer, lets say every '8' msecs.
655  * Either the user can set the 'tmo' or we can derive it based on
656  * a) line-speed and b) block-size.
657  * prb_calc_retire_blk_tmo() calculates the tmo.
658  *
659  */
660 static void prb_retire_rx_blk_timer_expired(unsigned long data)
661 {
662 	struct packet_sock *po = (struct packet_sock *)data;
663 	struct tpacket_kbdq_core *pkc = GET_PBDQC_FROM_RB(&po->rx_ring);
664 	unsigned int frozen;
665 	struct tpacket_block_desc *pbd;
666 
667 	spin_lock(&po->sk.sk_receive_queue.lock);
668 
669 	frozen = prb_queue_frozen(pkc);
670 	pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
671 
672 	if (unlikely(pkc->delete_blk_timer))
673 		goto out;
674 
675 	/* We only need to plug the race when the block is partially filled.
676 	 * tpacket_rcv:
677 	 *		lock(); increment BLOCK_NUM_PKTS; unlock()
678 	 *		copy_bits() is in progress ...
679 	 *		timer fires on other cpu:
680 	 *		we can't retire the current block because copy_bits
681 	 *		is in progress.
682 	 *
683 	 */
684 	if (BLOCK_NUM_PKTS(pbd)) {
685 		while (atomic_read(&pkc->blk_fill_in_prog)) {
686 			/* Waiting for skb_copy_bits to finish... */
687 			cpu_relax();
688 		}
689 	}
690 
691 	if (pkc->last_kactive_blk_num == pkc->kactive_blk_num) {
692 		if (!frozen) {
693 			prb_retire_current_block(pkc, po, TP_STATUS_BLK_TMO);
694 			if (!prb_dispatch_next_block(pkc, po))
695 				goto refresh_timer;
696 			else
697 				goto out;
698 		} else {
699 			/* Case 1. Queue was frozen because user-space was
700 			 *	   lagging behind.
701 			 */
702 			if (prb_curr_blk_in_use(pkc, pbd)) {
703 				/*
704 				 * Ok, user-space is still behind.
705 				 * So just refresh the timer.
706 				 */
707 				goto refresh_timer;
708 			} else {
709 			       /* Case 2. queue was frozen,user-space caught up,
710 				* now the link went idle && the timer fired.
711 				* We don't have a block to close.So we open this
712 				* block and restart the timer.
713 				* opening a block thaws the queue,restarts timer
714 				* Thawing/timer-refresh is a side effect.
715 				*/
716 				prb_open_block(pkc, pbd);
717 				goto out;
718 			}
719 		}
720 	}
721 
722 refresh_timer:
723 	_prb_refresh_rx_retire_blk_timer(pkc);
724 
725 out:
726 	spin_unlock(&po->sk.sk_receive_queue.lock);
727 }
728 
729 static void prb_flush_block(struct tpacket_kbdq_core *pkc1,
730 		struct tpacket_block_desc *pbd1, __u32 status)
731 {
732 	/* Flush everything minus the block header */
733 
734 #if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE == 1
735 	u8 *start, *end;
736 
737 	start = (u8 *)pbd1;
738 
739 	/* Skip the block header(we know header WILL fit in 4K) */
740 	start += PAGE_SIZE;
741 
742 	end = (u8 *)PAGE_ALIGN((unsigned long)pkc1->pkblk_end);
743 	for (; start < end; start += PAGE_SIZE)
744 		flush_dcache_page(pgv_to_page(start));
745 
746 	smp_wmb();
747 #endif
748 
749 	/* Now update the block status. */
750 
751 	BLOCK_STATUS(pbd1) = status;
752 
753 	/* Flush the block header */
754 
755 #if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE == 1
756 	start = (u8 *)pbd1;
757 	flush_dcache_page(pgv_to_page(start));
758 
759 	smp_wmb();
760 #endif
761 }
762 
763 /*
764  * Side effect:
765  *
766  * 1) flush the block
767  * 2) Increment active_blk_num
768  *
769  * Note:We DONT refresh the timer on purpose.
770  *	Because almost always the next block will be opened.
771  */
772 static void prb_close_block(struct tpacket_kbdq_core *pkc1,
773 		struct tpacket_block_desc *pbd1,
774 		struct packet_sock *po, unsigned int stat)
775 {
776 	__u32 status = TP_STATUS_USER | stat;
777 
778 	struct tpacket3_hdr *last_pkt;
779 	struct tpacket_hdr_v1 *h1 = &pbd1->hdr.bh1;
780 
781 	if (po->stats.stats3.tp_drops)
782 		status |= TP_STATUS_LOSING;
783 
784 	last_pkt = (struct tpacket3_hdr *)pkc1->prev;
785 	last_pkt->tp_next_offset = 0;
786 
787 	/* Get the ts of the last pkt */
788 	if (BLOCK_NUM_PKTS(pbd1)) {
789 		h1->ts_last_pkt.ts_sec = last_pkt->tp_sec;
790 		h1->ts_last_pkt.ts_nsec	= last_pkt->tp_nsec;
791 	} else {
792 		/* Ok, we tmo'd - so get the current time */
793 		struct timespec ts;
794 		getnstimeofday(&ts);
795 		h1->ts_last_pkt.ts_sec = ts.tv_sec;
796 		h1->ts_last_pkt.ts_nsec	= ts.tv_nsec;
797 	}
798 
799 	smp_wmb();
800 
801 	/* Flush the block */
802 	prb_flush_block(pkc1, pbd1, status);
803 
804 	pkc1->kactive_blk_num = GET_NEXT_PRB_BLK_NUM(pkc1);
805 }
806 
807 static void prb_thaw_queue(struct tpacket_kbdq_core *pkc)
808 {
809 	pkc->reset_pending_on_curr_blk = 0;
810 }
811 
812 /*
813  * Side effect of opening a block:
814  *
815  * 1) prb_queue is thawed.
816  * 2) retire_blk_timer is refreshed.
817  *
818  */
819 static void prb_open_block(struct tpacket_kbdq_core *pkc1,
820 	struct tpacket_block_desc *pbd1)
821 {
822 	struct timespec ts;
823 	struct tpacket_hdr_v1 *h1 = &pbd1->hdr.bh1;
824 
825 	smp_rmb();
826 
827 	/* We could have just memset this but we will lose the
828 	 * flexibility of making the priv area sticky
829 	 */
830 
831 	BLOCK_SNUM(pbd1) = pkc1->knxt_seq_num++;
832 	BLOCK_NUM_PKTS(pbd1) = 0;
833 	BLOCK_LEN(pbd1) = BLK_PLUS_PRIV(pkc1->blk_sizeof_priv);
834 
835 	getnstimeofday(&ts);
836 
837 	h1->ts_first_pkt.ts_sec = ts.tv_sec;
838 	h1->ts_first_pkt.ts_nsec = ts.tv_nsec;
839 
840 	pkc1->pkblk_start = (char *)pbd1;
841 	pkc1->nxt_offset = pkc1->pkblk_start + BLK_PLUS_PRIV(pkc1->blk_sizeof_priv);
842 
843 	BLOCK_O2FP(pbd1) = (__u32)BLK_PLUS_PRIV(pkc1->blk_sizeof_priv);
844 	BLOCK_O2PRIV(pbd1) = BLK_HDR_LEN;
845 
846 	pbd1->version = pkc1->version;
847 	pkc1->prev = pkc1->nxt_offset;
848 	pkc1->pkblk_end = pkc1->pkblk_start + pkc1->kblk_size;
849 
850 	prb_thaw_queue(pkc1);
851 	_prb_refresh_rx_retire_blk_timer(pkc1);
852 
853 	smp_wmb();
854 }
855 
856 /*
857  * Queue freeze logic:
858  * 1) Assume tp_block_nr = 8 blocks.
859  * 2) At time 't0', user opens Rx ring.
860  * 3) Some time past 't0', kernel starts filling blocks starting from 0 .. 7
861  * 4) user-space is either sleeping or processing block '0'.
862  * 5) tpacket_rcv is currently filling block '7', since there is no space left,
863  *    it will close block-7,loop around and try to fill block '0'.
864  *    call-flow:
865  *    __packet_lookup_frame_in_block
866  *      prb_retire_current_block()
867  *      prb_dispatch_next_block()
868  *        |->(BLOCK_STATUS == USER) evaluates to true
869  *    5.1) Since block-0 is currently in-use, we just freeze the queue.
870  * 6) Now there are two cases:
871  *    6.1) Link goes idle right after the queue is frozen.
872  *         But remember, the last open_block() refreshed the timer.
873  *         When this timer expires,it will refresh itself so that we can
874  *         re-open block-0 in near future.
875  *    6.2) Link is busy and keeps on receiving packets. This is a simple
876  *         case and __packet_lookup_frame_in_block will check if block-0
877  *         is free and can now be re-used.
878  */
879 static void prb_freeze_queue(struct tpacket_kbdq_core *pkc,
880 				  struct packet_sock *po)
881 {
882 	pkc->reset_pending_on_curr_blk = 1;
883 	po->stats.stats3.tp_freeze_q_cnt++;
884 }
885 
886 #define TOTAL_PKT_LEN_INCL_ALIGN(length) (ALIGN((length), V3_ALIGNMENT))
887 
888 /*
889  * If the next block is free then we will dispatch it
890  * and return a good offset.
891  * Else, we will freeze the queue.
892  * So, caller must check the return value.
893  */
894 static void *prb_dispatch_next_block(struct tpacket_kbdq_core *pkc,
895 		struct packet_sock *po)
896 {
897 	struct tpacket_block_desc *pbd;
898 
899 	smp_rmb();
900 
901 	/* 1. Get current block num */
902 	pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
903 
904 	/* 2. If this block is currently in_use then freeze the queue */
905 	if (TP_STATUS_USER & BLOCK_STATUS(pbd)) {
906 		prb_freeze_queue(pkc, po);
907 		return NULL;
908 	}
909 
910 	/*
911 	 * 3.
912 	 * open this block and return the offset where the first packet
913 	 * needs to get stored.
914 	 */
915 	prb_open_block(pkc, pbd);
916 	return (void *)pkc->nxt_offset;
917 }
918 
919 static void prb_retire_current_block(struct tpacket_kbdq_core *pkc,
920 		struct packet_sock *po, unsigned int status)
921 {
922 	struct tpacket_block_desc *pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
923 
924 	/* retire/close the current block */
925 	if (likely(TP_STATUS_KERNEL == BLOCK_STATUS(pbd))) {
926 		/*
927 		 * Plug the case where copy_bits() is in progress on
928 		 * cpu-0 and tpacket_rcv() got invoked on cpu-1, didn't
929 		 * have space to copy the pkt in the current block and
930 		 * called prb_retire_current_block()
931 		 *
932 		 * We don't need to worry about the TMO case because
933 		 * the timer-handler already handled this case.
934 		 */
935 		if (!(status & TP_STATUS_BLK_TMO)) {
936 			while (atomic_read(&pkc->blk_fill_in_prog)) {
937 				/* Waiting for skb_copy_bits to finish... */
938 				cpu_relax();
939 			}
940 		}
941 		prb_close_block(pkc, pbd, po, status);
942 		return;
943 	}
944 }
945 
946 static int prb_curr_blk_in_use(struct tpacket_kbdq_core *pkc,
947 				      struct tpacket_block_desc *pbd)
948 {
949 	return TP_STATUS_USER & BLOCK_STATUS(pbd);
950 }
951 
952 static int prb_queue_frozen(struct tpacket_kbdq_core *pkc)
953 {
954 	return pkc->reset_pending_on_curr_blk;
955 }
956 
957 static void prb_clear_blk_fill_status(struct packet_ring_buffer *rb)
958 {
959 	struct tpacket_kbdq_core *pkc  = GET_PBDQC_FROM_RB(rb);
960 	atomic_dec(&pkc->blk_fill_in_prog);
961 }
962 
963 static void prb_fill_rxhash(struct tpacket_kbdq_core *pkc,
964 			struct tpacket3_hdr *ppd)
965 {
966 	ppd->hv1.tp_rxhash = skb_get_hash(pkc->skb);
967 }
968 
969 static void prb_clear_rxhash(struct tpacket_kbdq_core *pkc,
970 			struct tpacket3_hdr *ppd)
971 {
972 	ppd->hv1.tp_rxhash = 0;
973 }
974 
975 static void prb_fill_vlan_info(struct tpacket_kbdq_core *pkc,
976 			struct tpacket3_hdr *ppd)
977 {
978 	if (vlan_tx_tag_present(pkc->skb)) {
979 		ppd->hv1.tp_vlan_tci = vlan_tx_tag_get(pkc->skb);
980 		ppd->hv1.tp_vlan_tpid = ntohs(pkc->skb->vlan_proto);
981 		ppd->tp_status = TP_STATUS_VLAN_VALID | TP_STATUS_VLAN_TPID_VALID;
982 	} else {
983 		ppd->hv1.tp_vlan_tci = 0;
984 		ppd->hv1.tp_vlan_tpid = 0;
985 		ppd->tp_status = TP_STATUS_AVAILABLE;
986 	}
987 }
988 
989 static void prb_run_all_ft_ops(struct tpacket_kbdq_core *pkc,
990 			struct tpacket3_hdr *ppd)
991 {
992 	ppd->hv1.tp_padding = 0;
993 	prb_fill_vlan_info(pkc, ppd);
994 
995 	if (pkc->feature_req_word & TP_FT_REQ_FILL_RXHASH)
996 		prb_fill_rxhash(pkc, ppd);
997 	else
998 		prb_clear_rxhash(pkc, ppd);
999 }
1000 
1001 static void prb_fill_curr_block(char *curr,
1002 				struct tpacket_kbdq_core *pkc,
1003 				struct tpacket_block_desc *pbd,
1004 				unsigned int len)
1005 {
1006 	struct tpacket3_hdr *ppd;
1007 
1008 	ppd  = (struct tpacket3_hdr *)curr;
1009 	ppd->tp_next_offset = TOTAL_PKT_LEN_INCL_ALIGN(len);
1010 	pkc->prev = curr;
1011 	pkc->nxt_offset += TOTAL_PKT_LEN_INCL_ALIGN(len);
1012 	BLOCK_LEN(pbd) += TOTAL_PKT_LEN_INCL_ALIGN(len);
1013 	BLOCK_NUM_PKTS(pbd) += 1;
1014 	atomic_inc(&pkc->blk_fill_in_prog);
1015 	prb_run_all_ft_ops(pkc, ppd);
1016 }
1017 
1018 /* Assumes caller has the sk->rx_queue.lock */
1019 static void *__packet_lookup_frame_in_block(struct packet_sock *po,
1020 					    struct sk_buff *skb,
1021 						int status,
1022 					    unsigned int len
1023 					    )
1024 {
1025 	struct tpacket_kbdq_core *pkc;
1026 	struct tpacket_block_desc *pbd;
1027 	char *curr, *end;
1028 
1029 	pkc = GET_PBDQC_FROM_RB(&po->rx_ring);
1030 	pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
1031 
1032 	/* Queue is frozen when user space is lagging behind */
1033 	if (prb_queue_frozen(pkc)) {
1034 		/*
1035 		 * Check if that last block which caused the queue to freeze,
1036 		 * is still in_use by user-space.
1037 		 */
1038 		if (prb_curr_blk_in_use(pkc, pbd)) {
1039 			/* Can't record this packet */
1040 			return NULL;
1041 		} else {
1042 			/*
1043 			 * Ok, the block was released by user-space.
1044 			 * Now let's open that block.
1045 			 * opening a block also thaws the queue.
1046 			 * Thawing is a side effect.
1047 			 */
1048 			prb_open_block(pkc, pbd);
1049 		}
1050 	}
1051 
1052 	smp_mb();
1053 	curr = pkc->nxt_offset;
1054 	pkc->skb = skb;
1055 	end = (char *)pbd + pkc->kblk_size;
1056 
1057 	/* first try the current block */
1058 	if (curr+TOTAL_PKT_LEN_INCL_ALIGN(len) < end) {
1059 		prb_fill_curr_block(curr, pkc, pbd, len);
1060 		return (void *)curr;
1061 	}
1062 
1063 	/* Ok, close the current block */
1064 	prb_retire_current_block(pkc, po, 0);
1065 
1066 	/* Now, try to dispatch the next block */
1067 	curr = (char *)prb_dispatch_next_block(pkc, po);
1068 	if (curr) {
1069 		pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
1070 		prb_fill_curr_block(curr, pkc, pbd, len);
1071 		return (void *)curr;
1072 	}
1073 
1074 	/*
1075 	 * No free blocks are available.user_space hasn't caught up yet.
1076 	 * Queue was just frozen and now this packet will get dropped.
1077 	 */
1078 	return NULL;
1079 }
1080 
1081 static void *packet_current_rx_frame(struct packet_sock *po,
1082 					    struct sk_buff *skb,
1083 					    int status, unsigned int len)
1084 {
1085 	char *curr = NULL;
1086 	switch (po->tp_version) {
1087 	case TPACKET_V1:
1088 	case TPACKET_V2:
1089 		curr = packet_lookup_frame(po, &po->rx_ring,
1090 					po->rx_ring.head, status);
1091 		return curr;
1092 	case TPACKET_V3:
1093 		return __packet_lookup_frame_in_block(po, skb, status, len);
1094 	default:
1095 		WARN(1, "TPACKET version not supported\n");
1096 		BUG();
1097 		return NULL;
1098 	}
1099 }
1100 
1101 static void *prb_lookup_block(struct packet_sock *po,
1102 				     struct packet_ring_buffer *rb,
1103 				     unsigned int idx,
1104 				     int status)
1105 {
1106 	struct tpacket_kbdq_core *pkc  = GET_PBDQC_FROM_RB(rb);
1107 	struct tpacket_block_desc *pbd = GET_PBLOCK_DESC(pkc, idx);
1108 
1109 	if (status != BLOCK_STATUS(pbd))
1110 		return NULL;
1111 	return pbd;
1112 }
1113 
1114 static int prb_previous_blk_num(struct packet_ring_buffer *rb)
1115 {
1116 	unsigned int prev;
1117 	if (rb->prb_bdqc.kactive_blk_num)
1118 		prev = rb->prb_bdqc.kactive_blk_num-1;
1119 	else
1120 		prev = rb->prb_bdqc.knum_blocks-1;
1121 	return prev;
1122 }
1123 
1124 /* Assumes caller has held the rx_queue.lock */
1125 static void *__prb_previous_block(struct packet_sock *po,
1126 					 struct packet_ring_buffer *rb,
1127 					 int status)
1128 {
1129 	unsigned int previous = prb_previous_blk_num(rb);
1130 	return prb_lookup_block(po, rb, previous, status);
1131 }
1132 
1133 static void *packet_previous_rx_frame(struct packet_sock *po,
1134 					     struct packet_ring_buffer *rb,
1135 					     int status)
1136 {
1137 	if (po->tp_version <= TPACKET_V2)
1138 		return packet_previous_frame(po, rb, status);
1139 
1140 	return __prb_previous_block(po, rb, status);
1141 }
1142 
1143 static void packet_increment_rx_head(struct packet_sock *po,
1144 					    struct packet_ring_buffer *rb)
1145 {
1146 	switch (po->tp_version) {
1147 	case TPACKET_V1:
1148 	case TPACKET_V2:
1149 		return packet_increment_head(rb);
1150 	case TPACKET_V3:
1151 	default:
1152 		WARN(1, "TPACKET version not supported.\n");
1153 		BUG();
1154 		return;
1155 	}
1156 }
1157 
1158 static void *packet_previous_frame(struct packet_sock *po,
1159 		struct packet_ring_buffer *rb,
1160 		int status)
1161 {
1162 	unsigned int previous = rb->head ? rb->head - 1 : rb->frame_max;
1163 	return packet_lookup_frame(po, rb, previous, status);
1164 }
1165 
1166 static void packet_increment_head(struct packet_ring_buffer *buff)
1167 {
1168 	buff->head = buff->head != buff->frame_max ? buff->head+1 : 0;
1169 }
1170 
1171 static void packet_inc_pending(struct packet_ring_buffer *rb)
1172 {
1173 	this_cpu_inc(*rb->pending_refcnt);
1174 }
1175 
1176 static void packet_dec_pending(struct packet_ring_buffer *rb)
1177 {
1178 	this_cpu_dec(*rb->pending_refcnt);
1179 }
1180 
1181 static unsigned int packet_read_pending(const struct packet_ring_buffer *rb)
1182 {
1183 	unsigned int refcnt = 0;
1184 	int cpu;
1185 
1186 	/* We don't use pending refcount in rx_ring. */
1187 	if (rb->pending_refcnt == NULL)
1188 		return 0;
1189 
1190 	for_each_possible_cpu(cpu)
1191 		refcnt += *per_cpu_ptr(rb->pending_refcnt, cpu);
1192 
1193 	return refcnt;
1194 }
1195 
1196 static int packet_alloc_pending(struct packet_sock *po)
1197 {
1198 	po->rx_ring.pending_refcnt = NULL;
1199 
1200 	po->tx_ring.pending_refcnt = alloc_percpu(unsigned int);
1201 	if (unlikely(po->tx_ring.pending_refcnt == NULL))
1202 		return -ENOBUFS;
1203 
1204 	return 0;
1205 }
1206 
1207 static void packet_free_pending(struct packet_sock *po)
1208 {
1209 	free_percpu(po->tx_ring.pending_refcnt);
1210 }
1211 
1212 static bool packet_rcv_has_room(struct packet_sock *po, struct sk_buff *skb)
1213 {
1214 	struct sock *sk = &po->sk;
1215 	bool has_room;
1216 
1217 	if (po->prot_hook.func != tpacket_rcv)
1218 		return (atomic_read(&sk->sk_rmem_alloc) + skb->truesize)
1219 			<= sk->sk_rcvbuf;
1220 
1221 	spin_lock(&sk->sk_receive_queue.lock);
1222 	if (po->tp_version == TPACKET_V3)
1223 		has_room = prb_lookup_block(po, &po->rx_ring,
1224 					    po->rx_ring.prb_bdqc.kactive_blk_num,
1225 					    TP_STATUS_KERNEL);
1226 	else
1227 		has_room = packet_lookup_frame(po, &po->rx_ring,
1228 					       po->rx_ring.head,
1229 					       TP_STATUS_KERNEL);
1230 	spin_unlock(&sk->sk_receive_queue.lock);
1231 
1232 	return has_room;
1233 }
1234 
1235 static void packet_sock_destruct(struct sock *sk)
1236 {
1237 	skb_queue_purge(&sk->sk_error_queue);
1238 
1239 	WARN_ON(atomic_read(&sk->sk_rmem_alloc));
1240 	WARN_ON(atomic_read(&sk->sk_wmem_alloc));
1241 
1242 	if (!sock_flag(sk, SOCK_DEAD)) {
1243 		pr_err("Attempt to release alive packet socket: %p\n", sk);
1244 		return;
1245 	}
1246 
1247 	sk_refcnt_debug_dec(sk);
1248 }
1249 
1250 static int fanout_rr_next(struct packet_fanout *f, unsigned int num)
1251 {
1252 	int x = atomic_read(&f->rr_cur) + 1;
1253 
1254 	if (x >= num)
1255 		x = 0;
1256 
1257 	return x;
1258 }
1259 
1260 static unsigned int fanout_demux_hash(struct packet_fanout *f,
1261 				      struct sk_buff *skb,
1262 				      unsigned int num)
1263 {
1264 	return reciprocal_scale(skb->rxhash, num);
1265 }
1266 
1267 static unsigned int fanout_demux_lb(struct packet_fanout *f,
1268 				    struct sk_buff *skb,
1269 				    unsigned int num)
1270 {
1271 	int cur, old;
1272 
1273 	cur = atomic_read(&f->rr_cur);
1274 	while ((old = atomic_cmpxchg(&f->rr_cur, cur,
1275 				     fanout_rr_next(f, num))) != cur)
1276 		cur = old;
1277 	return cur;
1278 }
1279 
1280 static unsigned int fanout_demux_cpu(struct packet_fanout *f,
1281 				     struct sk_buff *skb,
1282 				     unsigned int num)
1283 {
1284 	return smp_processor_id() % num;
1285 }
1286 
1287 static unsigned int fanout_demux_rnd(struct packet_fanout *f,
1288 				     struct sk_buff *skb,
1289 				     unsigned int num)
1290 {
1291 	return prandom_u32_max(num);
1292 }
1293 
1294 static unsigned int fanout_demux_rollover(struct packet_fanout *f,
1295 					  struct sk_buff *skb,
1296 					  unsigned int idx, unsigned int skip,
1297 					  unsigned int num)
1298 {
1299 	unsigned int i, j;
1300 
1301 	i = j = min_t(int, f->next[idx], num - 1);
1302 	do {
1303 		if (i != skip && packet_rcv_has_room(pkt_sk(f->arr[i]), skb)) {
1304 			if (i != j)
1305 				f->next[idx] = i;
1306 			return i;
1307 		}
1308 		if (++i == num)
1309 			i = 0;
1310 	} while (i != j);
1311 
1312 	return idx;
1313 }
1314 
1315 static unsigned int fanout_demux_qm(struct packet_fanout *f,
1316 				    struct sk_buff *skb,
1317 				    unsigned int num)
1318 {
1319 	return skb_get_queue_mapping(skb) % num;
1320 }
1321 
1322 static bool fanout_has_flag(struct packet_fanout *f, u16 flag)
1323 {
1324 	return f->flags & (flag >> 8);
1325 }
1326 
1327 static int packet_rcv_fanout(struct sk_buff *skb, struct net_device *dev,
1328 			     struct packet_type *pt, struct net_device *orig_dev)
1329 {
1330 	struct packet_fanout *f = pt->af_packet_priv;
1331 	unsigned int num = f->num_members;
1332 	struct packet_sock *po;
1333 	unsigned int idx;
1334 
1335 	if (!net_eq(dev_net(dev), read_pnet(&f->net)) ||
1336 	    !num) {
1337 		kfree_skb(skb);
1338 		return 0;
1339 	}
1340 
1341 	switch (f->type) {
1342 	case PACKET_FANOUT_HASH:
1343 	default:
1344 		if (fanout_has_flag(f, PACKET_FANOUT_FLAG_DEFRAG)) {
1345 			skb = ip_check_defrag(skb, IP_DEFRAG_AF_PACKET);
1346 			if (!skb)
1347 				return 0;
1348 		}
1349 		skb_get_hash(skb);
1350 		idx = fanout_demux_hash(f, skb, num);
1351 		break;
1352 	case PACKET_FANOUT_LB:
1353 		idx = fanout_demux_lb(f, skb, num);
1354 		break;
1355 	case PACKET_FANOUT_CPU:
1356 		idx = fanout_demux_cpu(f, skb, num);
1357 		break;
1358 	case PACKET_FANOUT_RND:
1359 		idx = fanout_demux_rnd(f, skb, num);
1360 		break;
1361 	case PACKET_FANOUT_QM:
1362 		idx = fanout_demux_qm(f, skb, num);
1363 		break;
1364 	case PACKET_FANOUT_ROLLOVER:
1365 		idx = fanout_demux_rollover(f, skb, 0, (unsigned int) -1, num);
1366 		break;
1367 	}
1368 
1369 	po = pkt_sk(f->arr[idx]);
1370 	if (fanout_has_flag(f, PACKET_FANOUT_FLAG_ROLLOVER) &&
1371 	    unlikely(!packet_rcv_has_room(po, skb))) {
1372 		idx = fanout_demux_rollover(f, skb, idx, idx, num);
1373 		po = pkt_sk(f->arr[idx]);
1374 	}
1375 
1376 	return po->prot_hook.func(skb, dev, &po->prot_hook, orig_dev);
1377 }
1378 
1379 DEFINE_MUTEX(fanout_mutex);
1380 EXPORT_SYMBOL_GPL(fanout_mutex);
1381 static LIST_HEAD(fanout_list);
1382 
1383 static void __fanout_link(struct sock *sk, struct packet_sock *po)
1384 {
1385 	struct packet_fanout *f = po->fanout;
1386 
1387 	spin_lock(&f->lock);
1388 	f->arr[f->num_members] = sk;
1389 	smp_wmb();
1390 	f->num_members++;
1391 	spin_unlock(&f->lock);
1392 }
1393 
1394 static void __fanout_unlink(struct sock *sk, struct packet_sock *po)
1395 {
1396 	struct packet_fanout *f = po->fanout;
1397 	int i;
1398 
1399 	spin_lock(&f->lock);
1400 	for (i = 0; i < f->num_members; i++) {
1401 		if (f->arr[i] == sk)
1402 			break;
1403 	}
1404 	BUG_ON(i >= f->num_members);
1405 	f->arr[i] = f->arr[f->num_members - 1];
1406 	f->num_members--;
1407 	spin_unlock(&f->lock);
1408 }
1409 
1410 static bool match_fanout_group(struct packet_type *ptype, struct sock *sk)
1411 {
1412 	if (ptype->af_packet_priv == (void *)((struct packet_sock *)sk)->fanout)
1413 		return true;
1414 
1415 	return false;
1416 }
1417 
1418 static int fanout_add(struct sock *sk, u16 id, u16 type_flags)
1419 {
1420 	struct packet_sock *po = pkt_sk(sk);
1421 	struct packet_fanout *f, *match;
1422 	u8 type = type_flags & 0xff;
1423 	u8 flags = type_flags >> 8;
1424 	int err;
1425 
1426 	switch (type) {
1427 	case PACKET_FANOUT_ROLLOVER:
1428 		if (type_flags & PACKET_FANOUT_FLAG_ROLLOVER)
1429 			return -EINVAL;
1430 	case PACKET_FANOUT_HASH:
1431 	case PACKET_FANOUT_LB:
1432 	case PACKET_FANOUT_CPU:
1433 	case PACKET_FANOUT_RND:
1434 	case PACKET_FANOUT_QM:
1435 		break;
1436 	default:
1437 		return -EINVAL;
1438 	}
1439 
1440 	if (!po->running)
1441 		return -EINVAL;
1442 
1443 	if (po->fanout)
1444 		return -EALREADY;
1445 
1446 	mutex_lock(&fanout_mutex);
1447 	match = NULL;
1448 	list_for_each_entry(f, &fanout_list, list) {
1449 		if (f->id == id &&
1450 		    read_pnet(&f->net) == sock_net(sk)) {
1451 			match = f;
1452 			break;
1453 		}
1454 	}
1455 	err = -EINVAL;
1456 	if (match && match->flags != flags)
1457 		goto out;
1458 	if (!match) {
1459 		err = -ENOMEM;
1460 		match = kzalloc(sizeof(*match), GFP_KERNEL);
1461 		if (!match)
1462 			goto out;
1463 		write_pnet(&match->net, sock_net(sk));
1464 		match->id = id;
1465 		match->type = type;
1466 		match->flags = flags;
1467 		atomic_set(&match->rr_cur, 0);
1468 		INIT_LIST_HEAD(&match->list);
1469 		spin_lock_init(&match->lock);
1470 		atomic_set(&match->sk_ref, 0);
1471 		match->prot_hook.type = po->prot_hook.type;
1472 		match->prot_hook.dev = po->prot_hook.dev;
1473 		match->prot_hook.func = packet_rcv_fanout;
1474 		match->prot_hook.af_packet_priv = match;
1475 		match->prot_hook.id_match = match_fanout_group;
1476 		dev_add_pack(&match->prot_hook);
1477 		list_add(&match->list, &fanout_list);
1478 	}
1479 	err = -EINVAL;
1480 	if (match->type == type &&
1481 	    match->prot_hook.type == po->prot_hook.type &&
1482 	    match->prot_hook.dev == po->prot_hook.dev) {
1483 		err = -ENOSPC;
1484 		if (atomic_read(&match->sk_ref) < PACKET_FANOUT_MAX) {
1485 			__dev_remove_pack(&po->prot_hook);
1486 			po->fanout = match;
1487 			atomic_inc(&match->sk_ref);
1488 			__fanout_link(sk, po);
1489 			err = 0;
1490 		}
1491 	}
1492 out:
1493 	mutex_unlock(&fanout_mutex);
1494 	return err;
1495 }
1496 
1497 static void fanout_release(struct sock *sk)
1498 {
1499 	struct packet_sock *po = pkt_sk(sk);
1500 	struct packet_fanout *f;
1501 
1502 	f = po->fanout;
1503 	if (!f)
1504 		return;
1505 
1506 	mutex_lock(&fanout_mutex);
1507 	po->fanout = NULL;
1508 
1509 	if (atomic_dec_and_test(&f->sk_ref)) {
1510 		list_del(&f->list);
1511 		dev_remove_pack(&f->prot_hook);
1512 		kfree(f);
1513 	}
1514 	mutex_unlock(&fanout_mutex);
1515 }
1516 
1517 static const struct proto_ops packet_ops;
1518 
1519 static const struct proto_ops packet_ops_spkt;
1520 
1521 static int packet_rcv_spkt(struct sk_buff *skb, struct net_device *dev,
1522 			   struct packet_type *pt, struct net_device *orig_dev)
1523 {
1524 	struct sock *sk;
1525 	struct sockaddr_pkt *spkt;
1526 
1527 	/*
1528 	 *	When we registered the protocol we saved the socket in the data
1529 	 *	field for just this event.
1530 	 */
1531 
1532 	sk = pt->af_packet_priv;
1533 
1534 	/*
1535 	 *	Yank back the headers [hope the device set this
1536 	 *	right or kerboom...]
1537 	 *
1538 	 *	Incoming packets have ll header pulled,
1539 	 *	push it back.
1540 	 *
1541 	 *	For outgoing ones skb->data == skb_mac_header(skb)
1542 	 *	so that this procedure is noop.
1543 	 */
1544 
1545 	if (skb->pkt_type == PACKET_LOOPBACK)
1546 		goto out;
1547 
1548 	if (!net_eq(dev_net(dev), sock_net(sk)))
1549 		goto out;
1550 
1551 	skb = skb_share_check(skb, GFP_ATOMIC);
1552 	if (skb == NULL)
1553 		goto oom;
1554 
1555 	/* drop any routing info */
1556 	skb_dst_drop(skb);
1557 
1558 	/* drop conntrack reference */
1559 	nf_reset(skb);
1560 
1561 	spkt = &PACKET_SKB_CB(skb)->sa.pkt;
1562 
1563 	skb_push(skb, skb->data - skb_mac_header(skb));
1564 
1565 	/*
1566 	 *	The SOCK_PACKET socket receives _all_ frames.
1567 	 */
1568 
1569 	spkt->spkt_family = dev->type;
1570 	strlcpy(spkt->spkt_device, dev->name, sizeof(spkt->spkt_device));
1571 	spkt->spkt_protocol = skb->protocol;
1572 
1573 	/*
1574 	 *	Charge the memory to the socket. This is done specifically
1575 	 *	to prevent sockets using all the memory up.
1576 	 */
1577 
1578 	if (sock_queue_rcv_skb(sk, skb) == 0)
1579 		return 0;
1580 
1581 out:
1582 	kfree_skb(skb);
1583 oom:
1584 	return 0;
1585 }
1586 
1587 
1588 /*
1589  *	Output a raw packet to a device layer. This bypasses all the other
1590  *	protocol layers and you must therefore supply it with a complete frame
1591  */
1592 
1593 static int packet_sendmsg_spkt(struct kiocb *iocb, struct socket *sock,
1594 			       struct msghdr *msg, size_t len)
1595 {
1596 	struct sock *sk = sock->sk;
1597 	DECLARE_SOCKADDR(struct sockaddr_pkt *, saddr, msg->msg_name);
1598 	struct sk_buff *skb = NULL;
1599 	struct net_device *dev;
1600 	__be16 proto = 0;
1601 	int err;
1602 	int extra_len = 0;
1603 
1604 	/*
1605 	 *	Get and verify the address.
1606 	 */
1607 
1608 	if (saddr) {
1609 		if (msg->msg_namelen < sizeof(struct sockaddr))
1610 			return -EINVAL;
1611 		if (msg->msg_namelen == sizeof(struct sockaddr_pkt))
1612 			proto = saddr->spkt_protocol;
1613 	} else
1614 		return -ENOTCONN;	/* SOCK_PACKET must be sent giving an address */
1615 
1616 	/*
1617 	 *	Find the device first to size check it
1618 	 */
1619 
1620 	saddr->spkt_device[sizeof(saddr->spkt_device) - 1] = 0;
1621 retry:
1622 	rcu_read_lock();
1623 	dev = dev_get_by_name_rcu(sock_net(sk), saddr->spkt_device);
1624 	err = -ENODEV;
1625 	if (dev == NULL)
1626 		goto out_unlock;
1627 
1628 	err = -ENETDOWN;
1629 	if (!(dev->flags & IFF_UP))
1630 		goto out_unlock;
1631 
1632 	/*
1633 	 * You may not queue a frame bigger than the mtu. This is the lowest level
1634 	 * raw protocol and you must do your own fragmentation at this level.
1635 	 */
1636 
1637 	if (unlikely(sock_flag(sk, SOCK_NOFCS))) {
1638 		if (!netif_supports_nofcs(dev)) {
1639 			err = -EPROTONOSUPPORT;
1640 			goto out_unlock;
1641 		}
1642 		extra_len = 4; /* We're doing our own CRC */
1643 	}
1644 
1645 	err = -EMSGSIZE;
1646 	if (len > dev->mtu + dev->hard_header_len + VLAN_HLEN + extra_len)
1647 		goto out_unlock;
1648 
1649 	if (!skb) {
1650 		size_t reserved = LL_RESERVED_SPACE(dev);
1651 		int tlen = dev->needed_tailroom;
1652 		unsigned int hhlen = dev->header_ops ? dev->hard_header_len : 0;
1653 
1654 		rcu_read_unlock();
1655 		skb = sock_wmalloc(sk, len + reserved + tlen, 0, GFP_KERNEL);
1656 		if (skb == NULL)
1657 			return -ENOBUFS;
1658 		/* FIXME: Save some space for broken drivers that write a hard
1659 		 * header at transmission time by themselves. PPP is the notable
1660 		 * one here. This should really be fixed at the driver level.
1661 		 */
1662 		skb_reserve(skb, reserved);
1663 		skb_reset_network_header(skb);
1664 
1665 		/* Try to align data part correctly */
1666 		if (hhlen) {
1667 			skb->data -= hhlen;
1668 			skb->tail -= hhlen;
1669 			if (len < hhlen)
1670 				skb_reset_network_header(skb);
1671 		}
1672 		err = memcpy_fromiovec(skb_put(skb, len), msg->msg_iov, len);
1673 		if (err)
1674 			goto out_free;
1675 		goto retry;
1676 	}
1677 
1678 	if (len > (dev->mtu + dev->hard_header_len + extra_len)) {
1679 		/* Earlier code assumed this would be a VLAN pkt,
1680 		 * double-check this now that we have the actual
1681 		 * packet in hand.
1682 		 */
1683 		struct ethhdr *ehdr;
1684 		skb_reset_mac_header(skb);
1685 		ehdr = eth_hdr(skb);
1686 		if (ehdr->h_proto != htons(ETH_P_8021Q)) {
1687 			err = -EMSGSIZE;
1688 			goto out_unlock;
1689 		}
1690 	}
1691 
1692 	skb->protocol = proto;
1693 	skb->dev = dev;
1694 	skb->priority = sk->sk_priority;
1695 	skb->mark = sk->sk_mark;
1696 
1697 	sock_tx_timestamp(sk, &skb_shinfo(skb)->tx_flags);
1698 
1699 	if (unlikely(extra_len == 4))
1700 		skb->no_fcs = 1;
1701 
1702 	skb_probe_transport_header(skb, 0);
1703 
1704 	dev_queue_xmit(skb);
1705 	rcu_read_unlock();
1706 	return len;
1707 
1708 out_unlock:
1709 	rcu_read_unlock();
1710 out_free:
1711 	kfree_skb(skb);
1712 	return err;
1713 }
1714 
1715 static unsigned int run_filter(const struct sk_buff *skb,
1716 				      const struct sock *sk,
1717 				      unsigned int res)
1718 {
1719 	struct sk_filter *filter;
1720 
1721 	rcu_read_lock();
1722 	filter = rcu_dereference(sk->sk_filter);
1723 	if (filter != NULL)
1724 		res = SK_RUN_FILTER(filter, skb);
1725 	rcu_read_unlock();
1726 
1727 	return res;
1728 }
1729 
1730 /*
1731  * This function makes lazy skb cloning in hope that most of packets
1732  * are discarded by BPF.
1733  *
1734  * Note tricky part: we DO mangle shared skb! skb->data, skb->len
1735  * and skb->cb are mangled. It works because (and until) packets
1736  * falling here are owned by current CPU. Output packets are cloned
1737  * by dev_queue_xmit_nit(), input packets are processed by net_bh
1738  * sequencially, so that if we return skb to original state on exit,
1739  * we will not harm anyone.
1740  */
1741 
1742 static int packet_rcv(struct sk_buff *skb, struct net_device *dev,
1743 		      struct packet_type *pt, struct net_device *orig_dev)
1744 {
1745 	struct sock *sk;
1746 	struct sockaddr_ll *sll;
1747 	struct packet_sock *po;
1748 	u8 *skb_head = skb->data;
1749 	int skb_len = skb->len;
1750 	unsigned int snaplen, res;
1751 
1752 	if (skb->pkt_type == PACKET_LOOPBACK)
1753 		goto drop;
1754 
1755 	sk = pt->af_packet_priv;
1756 	po = pkt_sk(sk);
1757 
1758 	if (!net_eq(dev_net(dev), sock_net(sk)))
1759 		goto drop;
1760 
1761 	skb->dev = dev;
1762 
1763 	if (dev->header_ops) {
1764 		/* The device has an explicit notion of ll header,
1765 		 * exported to higher levels.
1766 		 *
1767 		 * Otherwise, the device hides details of its frame
1768 		 * structure, so that corresponding packet head is
1769 		 * never delivered to user.
1770 		 */
1771 		if (sk->sk_type != SOCK_DGRAM)
1772 			skb_push(skb, skb->data - skb_mac_header(skb));
1773 		else if (skb->pkt_type == PACKET_OUTGOING) {
1774 			/* Special case: outgoing packets have ll header at head */
1775 			skb_pull(skb, skb_network_offset(skb));
1776 		}
1777 	}
1778 
1779 	snaplen = skb->len;
1780 
1781 	res = run_filter(skb, sk, snaplen);
1782 	if (!res)
1783 		goto drop_n_restore;
1784 	if (snaplen > res)
1785 		snaplen = res;
1786 
1787 	if (atomic_read(&sk->sk_rmem_alloc) >= sk->sk_rcvbuf)
1788 		goto drop_n_acct;
1789 
1790 	if (skb_shared(skb)) {
1791 		struct sk_buff *nskb = skb_clone(skb, GFP_ATOMIC);
1792 		if (nskb == NULL)
1793 			goto drop_n_acct;
1794 
1795 		if (skb_head != skb->data) {
1796 			skb->data = skb_head;
1797 			skb->len = skb_len;
1798 		}
1799 		consume_skb(skb);
1800 		skb = nskb;
1801 	}
1802 
1803 	BUILD_BUG_ON(sizeof(*PACKET_SKB_CB(skb)) + MAX_ADDR_LEN - 8 >
1804 		     sizeof(skb->cb));
1805 
1806 	sll = &PACKET_SKB_CB(skb)->sa.ll;
1807 	sll->sll_family = AF_PACKET;
1808 	sll->sll_hatype = dev->type;
1809 	sll->sll_protocol = skb->protocol;
1810 	sll->sll_pkttype = skb->pkt_type;
1811 	if (unlikely(po->origdev))
1812 		sll->sll_ifindex = orig_dev->ifindex;
1813 	else
1814 		sll->sll_ifindex = dev->ifindex;
1815 
1816 	sll->sll_halen = dev_parse_header(skb, sll->sll_addr);
1817 
1818 	PACKET_SKB_CB(skb)->origlen = skb->len;
1819 
1820 	if (pskb_trim(skb, snaplen))
1821 		goto drop_n_acct;
1822 
1823 	skb_set_owner_r(skb, sk);
1824 	skb->dev = NULL;
1825 	skb_dst_drop(skb);
1826 
1827 	/* drop conntrack reference */
1828 	nf_reset(skb);
1829 
1830 	spin_lock(&sk->sk_receive_queue.lock);
1831 	po->stats.stats1.tp_packets++;
1832 	skb->dropcount = atomic_read(&sk->sk_drops);
1833 	__skb_queue_tail(&sk->sk_receive_queue, skb);
1834 	spin_unlock(&sk->sk_receive_queue.lock);
1835 	sk->sk_data_ready(sk, skb->len);
1836 	return 0;
1837 
1838 drop_n_acct:
1839 	spin_lock(&sk->sk_receive_queue.lock);
1840 	po->stats.stats1.tp_drops++;
1841 	atomic_inc(&sk->sk_drops);
1842 	spin_unlock(&sk->sk_receive_queue.lock);
1843 
1844 drop_n_restore:
1845 	if (skb_head != skb->data && skb_shared(skb)) {
1846 		skb->data = skb_head;
1847 		skb->len = skb_len;
1848 	}
1849 drop:
1850 	consume_skb(skb);
1851 	return 0;
1852 }
1853 
1854 static int tpacket_rcv(struct sk_buff *skb, struct net_device *dev,
1855 		       struct packet_type *pt, struct net_device *orig_dev)
1856 {
1857 	struct sock *sk;
1858 	struct packet_sock *po;
1859 	struct sockaddr_ll *sll;
1860 	union tpacket_uhdr h;
1861 	u8 *skb_head = skb->data;
1862 	int skb_len = skb->len;
1863 	unsigned int snaplen, res;
1864 	unsigned long status = TP_STATUS_USER;
1865 	unsigned short macoff, netoff, hdrlen;
1866 	struct sk_buff *copy_skb = NULL;
1867 	struct timespec ts;
1868 	__u32 ts_status;
1869 
1870 	/* struct tpacket{2,3}_hdr is aligned to a multiple of TPACKET_ALIGNMENT.
1871 	 * We may add members to them until current aligned size without forcing
1872 	 * userspace to call getsockopt(..., PACKET_HDRLEN, ...).
1873 	 */
1874 	BUILD_BUG_ON(TPACKET_ALIGN(sizeof(*h.h2)) != 32);
1875 	BUILD_BUG_ON(TPACKET_ALIGN(sizeof(*h.h3)) != 48);
1876 
1877 	if (skb->pkt_type == PACKET_LOOPBACK)
1878 		goto drop;
1879 
1880 	sk = pt->af_packet_priv;
1881 	po = pkt_sk(sk);
1882 
1883 	if (!net_eq(dev_net(dev), sock_net(sk)))
1884 		goto drop;
1885 
1886 	if (dev->header_ops) {
1887 		if (sk->sk_type != SOCK_DGRAM)
1888 			skb_push(skb, skb->data - skb_mac_header(skb));
1889 		else if (skb->pkt_type == PACKET_OUTGOING) {
1890 			/* Special case: outgoing packets have ll header at head */
1891 			skb_pull(skb, skb_network_offset(skb));
1892 		}
1893 	}
1894 
1895 	if (skb->ip_summed == CHECKSUM_PARTIAL)
1896 		status |= TP_STATUS_CSUMNOTREADY;
1897 
1898 	snaplen = skb->len;
1899 
1900 	res = run_filter(skb, sk, snaplen);
1901 	if (!res)
1902 		goto drop_n_restore;
1903 	if (snaplen > res)
1904 		snaplen = res;
1905 
1906 	if (sk->sk_type == SOCK_DGRAM) {
1907 		macoff = netoff = TPACKET_ALIGN(po->tp_hdrlen) + 16 +
1908 				  po->tp_reserve;
1909 	} else {
1910 		unsigned int maclen = skb_network_offset(skb);
1911 		netoff = TPACKET_ALIGN(po->tp_hdrlen +
1912 				       (maclen < 16 ? 16 : maclen)) +
1913 			po->tp_reserve;
1914 		macoff = netoff - maclen;
1915 	}
1916 	if (po->tp_version <= TPACKET_V2) {
1917 		if (macoff + snaplen > po->rx_ring.frame_size) {
1918 			if (po->copy_thresh &&
1919 			    atomic_read(&sk->sk_rmem_alloc) < sk->sk_rcvbuf) {
1920 				if (skb_shared(skb)) {
1921 					copy_skb = skb_clone(skb, GFP_ATOMIC);
1922 				} else {
1923 					copy_skb = skb_get(skb);
1924 					skb_head = skb->data;
1925 				}
1926 				if (copy_skb)
1927 					skb_set_owner_r(copy_skb, sk);
1928 			}
1929 			snaplen = po->rx_ring.frame_size - macoff;
1930 			if ((int)snaplen < 0)
1931 				snaplen = 0;
1932 		}
1933 	}
1934 	spin_lock(&sk->sk_receive_queue.lock);
1935 	h.raw = packet_current_rx_frame(po, skb,
1936 					TP_STATUS_KERNEL, (macoff+snaplen));
1937 	if (!h.raw)
1938 		goto ring_is_full;
1939 	if (po->tp_version <= TPACKET_V2) {
1940 		packet_increment_rx_head(po, &po->rx_ring);
1941 	/*
1942 	 * LOSING will be reported till you read the stats,
1943 	 * because it's COR - Clear On Read.
1944 	 * Anyways, moving it for V1/V2 only as V3 doesn't need this
1945 	 * at packet level.
1946 	 */
1947 		if (po->stats.stats1.tp_drops)
1948 			status |= TP_STATUS_LOSING;
1949 	}
1950 	po->stats.stats1.tp_packets++;
1951 	if (copy_skb) {
1952 		status |= TP_STATUS_COPY;
1953 		__skb_queue_tail(&sk->sk_receive_queue, copy_skb);
1954 	}
1955 	spin_unlock(&sk->sk_receive_queue.lock);
1956 
1957 	skb_copy_bits(skb, 0, h.raw + macoff, snaplen);
1958 
1959 	if (!(ts_status = tpacket_get_timestamp(skb, &ts, po->tp_tstamp)))
1960 		getnstimeofday(&ts);
1961 
1962 	status |= ts_status;
1963 
1964 	switch (po->tp_version) {
1965 	case TPACKET_V1:
1966 		h.h1->tp_len = skb->len;
1967 		h.h1->tp_snaplen = snaplen;
1968 		h.h1->tp_mac = macoff;
1969 		h.h1->tp_net = netoff;
1970 		h.h1->tp_sec = ts.tv_sec;
1971 		h.h1->tp_usec = ts.tv_nsec / NSEC_PER_USEC;
1972 		hdrlen = sizeof(*h.h1);
1973 		break;
1974 	case TPACKET_V2:
1975 		h.h2->tp_len = skb->len;
1976 		h.h2->tp_snaplen = snaplen;
1977 		h.h2->tp_mac = macoff;
1978 		h.h2->tp_net = netoff;
1979 		h.h2->tp_sec = ts.tv_sec;
1980 		h.h2->tp_nsec = ts.tv_nsec;
1981 		if (vlan_tx_tag_present(skb)) {
1982 			h.h2->tp_vlan_tci = vlan_tx_tag_get(skb);
1983 			h.h2->tp_vlan_tpid = ntohs(skb->vlan_proto);
1984 			status |= TP_STATUS_VLAN_VALID | TP_STATUS_VLAN_TPID_VALID;
1985 		} else {
1986 			h.h2->tp_vlan_tci = 0;
1987 			h.h2->tp_vlan_tpid = 0;
1988 		}
1989 		memset(h.h2->tp_padding, 0, sizeof(h.h2->tp_padding));
1990 		hdrlen = sizeof(*h.h2);
1991 		break;
1992 	case TPACKET_V3:
1993 		/* tp_nxt_offset,vlan are already populated above.
1994 		 * So DONT clear those fields here
1995 		 */
1996 		h.h3->tp_status |= status;
1997 		h.h3->tp_len = skb->len;
1998 		h.h3->tp_snaplen = snaplen;
1999 		h.h3->tp_mac = macoff;
2000 		h.h3->tp_net = netoff;
2001 		h.h3->tp_sec  = ts.tv_sec;
2002 		h.h3->tp_nsec = ts.tv_nsec;
2003 		memset(h.h3->tp_padding, 0, sizeof(h.h3->tp_padding));
2004 		hdrlen = sizeof(*h.h3);
2005 		break;
2006 	default:
2007 		BUG();
2008 	}
2009 
2010 	sll = h.raw + TPACKET_ALIGN(hdrlen);
2011 	sll->sll_halen = dev_parse_header(skb, sll->sll_addr);
2012 	sll->sll_family = AF_PACKET;
2013 	sll->sll_hatype = dev->type;
2014 	sll->sll_protocol = skb->protocol;
2015 	sll->sll_pkttype = skb->pkt_type;
2016 	if (unlikely(po->origdev))
2017 		sll->sll_ifindex = orig_dev->ifindex;
2018 	else
2019 		sll->sll_ifindex = dev->ifindex;
2020 
2021 	smp_mb();
2022 
2023 #if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE == 1
2024 	if (po->tp_version <= TPACKET_V2) {
2025 		u8 *start, *end;
2026 
2027 		end = (u8 *) PAGE_ALIGN((unsigned long) h.raw +
2028 					macoff + snaplen);
2029 
2030 		for (start = h.raw; start < end; start += PAGE_SIZE)
2031 			flush_dcache_page(pgv_to_page(start));
2032 	}
2033 	smp_wmb();
2034 #endif
2035 
2036 	if (po->tp_version <= TPACKET_V2)
2037 		__packet_set_status(po, h.raw, status);
2038 	else
2039 		prb_clear_blk_fill_status(&po->rx_ring);
2040 
2041 	sk->sk_data_ready(sk, 0);
2042 
2043 drop_n_restore:
2044 	if (skb_head != skb->data && skb_shared(skb)) {
2045 		skb->data = skb_head;
2046 		skb->len = skb_len;
2047 	}
2048 drop:
2049 	kfree_skb(skb);
2050 	return 0;
2051 
2052 ring_is_full:
2053 	po->stats.stats1.tp_drops++;
2054 	spin_unlock(&sk->sk_receive_queue.lock);
2055 
2056 	sk->sk_data_ready(sk, 0);
2057 	kfree_skb(copy_skb);
2058 	goto drop_n_restore;
2059 }
2060 
2061 static void tpacket_destruct_skb(struct sk_buff *skb)
2062 {
2063 	struct packet_sock *po = pkt_sk(skb->sk);
2064 
2065 	if (likely(po->tx_ring.pg_vec)) {
2066 		void *ph;
2067 		__u32 ts;
2068 
2069 		ph = skb_shinfo(skb)->destructor_arg;
2070 		packet_dec_pending(&po->tx_ring);
2071 
2072 		ts = __packet_set_timestamp(po, ph, skb);
2073 		__packet_set_status(po, ph, TP_STATUS_AVAILABLE | ts);
2074 	}
2075 
2076 	sock_wfree(skb);
2077 }
2078 
2079 static int tpacket_fill_skb(struct packet_sock *po, struct sk_buff *skb,
2080 		void *frame, struct net_device *dev, int size_max,
2081 		__be16 proto, unsigned char *addr, int hlen)
2082 {
2083 	union tpacket_uhdr ph;
2084 	int to_write, offset, len, tp_len, nr_frags, len_max;
2085 	struct socket *sock = po->sk.sk_socket;
2086 	struct page *page;
2087 	void *data;
2088 	int err;
2089 
2090 	ph.raw = frame;
2091 
2092 	skb->protocol = proto;
2093 	skb->dev = dev;
2094 	skb->priority = po->sk.sk_priority;
2095 	skb->mark = po->sk.sk_mark;
2096 	sock_tx_timestamp(&po->sk, &skb_shinfo(skb)->tx_flags);
2097 	skb_shinfo(skb)->destructor_arg = ph.raw;
2098 
2099 	switch (po->tp_version) {
2100 	case TPACKET_V2:
2101 		tp_len = ph.h2->tp_len;
2102 		break;
2103 	default:
2104 		tp_len = ph.h1->tp_len;
2105 		break;
2106 	}
2107 	if (unlikely(tp_len > size_max)) {
2108 		pr_err("packet size is too long (%d > %d)\n", tp_len, size_max);
2109 		return -EMSGSIZE;
2110 	}
2111 
2112 	skb_reserve(skb, hlen);
2113 	skb_reset_network_header(skb);
2114 
2115 	if (!packet_use_direct_xmit(po))
2116 		skb_probe_transport_header(skb, 0);
2117 	if (unlikely(po->tp_tx_has_off)) {
2118 		int off_min, off_max, off;
2119 		off_min = po->tp_hdrlen - sizeof(struct sockaddr_ll);
2120 		off_max = po->tx_ring.frame_size - tp_len;
2121 		if (sock->type == SOCK_DGRAM) {
2122 			switch (po->tp_version) {
2123 			case TPACKET_V2:
2124 				off = ph.h2->tp_net;
2125 				break;
2126 			default:
2127 				off = ph.h1->tp_net;
2128 				break;
2129 			}
2130 		} else {
2131 			switch (po->tp_version) {
2132 			case TPACKET_V2:
2133 				off = ph.h2->tp_mac;
2134 				break;
2135 			default:
2136 				off = ph.h1->tp_mac;
2137 				break;
2138 			}
2139 		}
2140 		if (unlikely((off < off_min) || (off_max < off)))
2141 			return -EINVAL;
2142 		data = ph.raw + off;
2143 	} else {
2144 		data = ph.raw + po->tp_hdrlen - sizeof(struct sockaddr_ll);
2145 	}
2146 	to_write = tp_len;
2147 
2148 	if (sock->type == SOCK_DGRAM) {
2149 		err = dev_hard_header(skb, dev, ntohs(proto), addr,
2150 				NULL, tp_len);
2151 		if (unlikely(err < 0))
2152 			return -EINVAL;
2153 	} else if (dev->hard_header_len) {
2154 		/* net device doesn't like empty head */
2155 		if (unlikely(tp_len <= dev->hard_header_len)) {
2156 			pr_err("packet size is too short (%d < %d)\n",
2157 			       tp_len, dev->hard_header_len);
2158 			return -EINVAL;
2159 		}
2160 
2161 		skb_push(skb, dev->hard_header_len);
2162 		err = skb_store_bits(skb, 0, data,
2163 				dev->hard_header_len);
2164 		if (unlikely(err))
2165 			return err;
2166 
2167 		data += dev->hard_header_len;
2168 		to_write -= dev->hard_header_len;
2169 	}
2170 
2171 	offset = offset_in_page(data);
2172 	len_max = PAGE_SIZE - offset;
2173 	len = ((to_write > len_max) ? len_max : to_write);
2174 
2175 	skb->data_len = to_write;
2176 	skb->len += to_write;
2177 	skb->truesize += to_write;
2178 	atomic_add(to_write, &po->sk.sk_wmem_alloc);
2179 
2180 	while (likely(to_write)) {
2181 		nr_frags = skb_shinfo(skb)->nr_frags;
2182 
2183 		if (unlikely(nr_frags >= MAX_SKB_FRAGS)) {
2184 			pr_err("Packet exceed the number of skb frags(%lu)\n",
2185 			       MAX_SKB_FRAGS);
2186 			return -EFAULT;
2187 		}
2188 
2189 		page = pgv_to_page(data);
2190 		data += len;
2191 		flush_dcache_page(page);
2192 		get_page(page);
2193 		skb_fill_page_desc(skb, nr_frags, page, offset, len);
2194 		to_write -= len;
2195 		offset = 0;
2196 		len_max = PAGE_SIZE;
2197 		len = ((to_write > len_max) ? len_max : to_write);
2198 	}
2199 
2200 	return tp_len;
2201 }
2202 
2203 static int tpacket_snd(struct packet_sock *po, struct msghdr *msg)
2204 {
2205 	struct sk_buff *skb;
2206 	struct net_device *dev;
2207 	__be16 proto;
2208 	int err, reserve = 0;
2209 	void *ph;
2210 	DECLARE_SOCKADDR(struct sockaddr_ll *, saddr, msg->msg_name);
2211 	bool need_wait = !(msg->msg_flags & MSG_DONTWAIT);
2212 	int tp_len, size_max;
2213 	unsigned char *addr;
2214 	int len_sum = 0;
2215 	int status = TP_STATUS_AVAILABLE;
2216 	int hlen, tlen;
2217 
2218 	mutex_lock(&po->pg_vec_lock);
2219 
2220 	if (likely(saddr == NULL)) {
2221 		dev	= packet_cached_dev_get(po);
2222 		proto	= po->num;
2223 		addr	= NULL;
2224 	} else {
2225 		err = -EINVAL;
2226 		if (msg->msg_namelen < sizeof(struct sockaddr_ll))
2227 			goto out;
2228 		if (msg->msg_namelen < (saddr->sll_halen
2229 					+ offsetof(struct sockaddr_ll,
2230 						sll_addr)))
2231 			goto out;
2232 		proto	= saddr->sll_protocol;
2233 		addr	= saddr->sll_addr;
2234 		dev = dev_get_by_index(sock_net(&po->sk), saddr->sll_ifindex);
2235 	}
2236 
2237 	err = -ENXIO;
2238 	if (unlikely(dev == NULL))
2239 		goto out;
2240 	err = -ENETDOWN;
2241 	if (unlikely(!(dev->flags & IFF_UP)))
2242 		goto out_put;
2243 
2244 	reserve = dev->hard_header_len;
2245 
2246 	size_max = po->tx_ring.frame_size
2247 		- (po->tp_hdrlen - sizeof(struct sockaddr_ll));
2248 
2249 	if (size_max > dev->mtu + reserve)
2250 		size_max = dev->mtu + reserve;
2251 
2252 	do {
2253 		ph = packet_current_frame(po, &po->tx_ring,
2254 					  TP_STATUS_SEND_REQUEST);
2255 		if (unlikely(ph == NULL)) {
2256 			if (need_wait && need_resched())
2257 				schedule();
2258 			continue;
2259 		}
2260 
2261 		status = TP_STATUS_SEND_REQUEST;
2262 		hlen = LL_RESERVED_SPACE(dev);
2263 		tlen = dev->needed_tailroom;
2264 		skb = sock_alloc_send_skb(&po->sk,
2265 				hlen + tlen + sizeof(struct sockaddr_ll),
2266 				0, &err);
2267 
2268 		if (unlikely(skb == NULL))
2269 			goto out_status;
2270 
2271 		tp_len = tpacket_fill_skb(po, skb, ph, dev, size_max, proto,
2272 				addr, hlen);
2273 
2274 		if (unlikely(tp_len < 0)) {
2275 			if (po->tp_loss) {
2276 				__packet_set_status(po, ph,
2277 						TP_STATUS_AVAILABLE);
2278 				packet_increment_head(&po->tx_ring);
2279 				kfree_skb(skb);
2280 				continue;
2281 			} else {
2282 				status = TP_STATUS_WRONG_FORMAT;
2283 				err = tp_len;
2284 				goto out_status;
2285 			}
2286 		}
2287 
2288 		skb_set_queue_mapping(skb, packet_pick_tx_queue(dev));
2289 		skb->destructor = tpacket_destruct_skb;
2290 		__packet_set_status(po, ph, TP_STATUS_SENDING);
2291 		packet_inc_pending(&po->tx_ring);
2292 
2293 		status = TP_STATUS_SEND_REQUEST;
2294 		err = po->xmit(skb);
2295 		if (unlikely(err > 0)) {
2296 			err = net_xmit_errno(err);
2297 			if (err && __packet_get_status(po, ph) ==
2298 				   TP_STATUS_AVAILABLE) {
2299 				/* skb was destructed already */
2300 				skb = NULL;
2301 				goto out_status;
2302 			}
2303 			/*
2304 			 * skb was dropped but not destructed yet;
2305 			 * let's treat it like congestion or err < 0
2306 			 */
2307 			err = 0;
2308 		}
2309 		packet_increment_head(&po->tx_ring);
2310 		len_sum += tp_len;
2311 	} while (likely((ph != NULL) ||
2312 		/* Note: packet_read_pending() might be slow if we have
2313 		 * to call it as it's per_cpu variable, but in fast-path
2314 		 * we already short-circuit the loop with the first
2315 		 * condition, and luckily don't have to go that path
2316 		 * anyway.
2317 		 */
2318 		 (need_wait && packet_read_pending(&po->tx_ring))));
2319 
2320 	err = len_sum;
2321 	goto out_put;
2322 
2323 out_status:
2324 	__packet_set_status(po, ph, status);
2325 	kfree_skb(skb);
2326 out_put:
2327 	dev_put(dev);
2328 out:
2329 	mutex_unlock(&po->pg_vec_lock);
2330 	return err;
2331 }
2332 
2333 static struct sk_buff *packet_alloc_skb(struct sock *sk, size_t prepad,
2334 				        size_t reserve, size_t len,
2335 				        size_t linear, int noblock,
2336 				        int *err)
2337 {
2338 	struct sk_buff *skb;
2339 
2340 	/* Under a page?  Don't bother with paged skb. */
2341 	if (prepad + len < PAGE_SIZE || !linear)
2342 		linear = len;
2343 
2344 	skb = sock_alloc_send_pskb(sk, prepad + linear, len - linear, noblock,
2345 				   err, 0);
2346 	if (!skb)
2347 		return NULL;
2348 
2349 	skb_reserve(skb, reserve);
2350 	skb_put(skb, linear);
2351 	skb->data_len = len - linear;
2352 	skb->len += len - linear;
2353 
2354 	return skb;
2355 }
2356 
2357 static int packet_snd(struct socket *sock, struct msghdr *msg, size_t len)
2358 {
2359 	struct sock *sk = sock->sk;
2360 	DECLARE_SOCKADDR(struct sockaddr_ll *, saddr, msg->msg_name);
2361 	struct sk_buff *skb;
2362 	struct net_device *dev;
2363 	__be16 proto;
2364 	unsigned char *addr;
2365 	int err, reserve = 0;
2366 	struct virtio_net_hdr vnet_hdr = { 0 };
2367 	int offset = 0;
2368 	int vnet_hdr_len;
2369 	struct packet_sock *po = pkt_sk(sk);
2370 	unsigned short gso_type = 0;
2371 	int hlen, tlen;
2372 	int extra_len = 0;
2373 
2374 	/*
2375 	 *	Get and verify the address.
2376 	 */
2377 
2378 	if (likely(saddr == NULL)) {
2379 		dev	= packet_cached_dev_get(po);
2380 		proto	= po->num;
2381 		addr	= NULL;
2382 	} else {
2383 		err = -EINVAL;
2384 		if (msg->msg_namelen < sizeof(struct sockaddr_ll))
2385 			goto out;
2386 		if (msg->msg_namelen < (saddr->sll_halen + offsetof(struct sockaddr_ll, sll_addr)))
2387 			goto out;
2388 		proto	= saddr->sll_protocol;
2389 		addr	= saddr->sll_addr;
2390 		dev = dev_get_by_index(sock_net(sk), saddr->sll_ifindex);
2391 	}
2392 
2393 	err = -ENXIO;
2394 	if (unlikely(dev == NULL))
2395 		goto out_unlock;
2396 	err = -ENETDOWN;
2397 	if (unlikely(!(dev->flags & IFF_UP)))
2398 		goto out_unlock;
2399 
2400 	if (sock->type == SOCK_RAW)
2401 		reserve = dev->hard_header_len;
2402 	if (po->has_vnet_hdr) {
2403 		vnet_hdr_len = sizeof(vnet_hdr);
2404 
2405 		err = -EINVAL;
2406 		if (len < vnet_hdr_len)
2407 			goto out_unlock;
2408 
2409 		len -= vnet_hdr_len;
2410 
2411 		err = memcpy_fromiovec((void *)&vnet_hdr, msg->msg_iov,
2412 				       vnet_hdr_len);
2413 		if (err < 0)
2414 			goto out_unlock;
2415 
2416 		if ((vnet_hdr.flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) &&
2417 		    (vnet_hdr.csum_start + vnet_hdr.csum_offset + 2 >
2418 		      vnet_hdr.hdr_len))
2419 			vnet_hdr.hdr_len = vnet_hdr.csum_start +
2420 						 vnet_hdr.csum_offset + 2;
2421 
2422 		err = -EINVAL;
2423 		if (vnet_hdr.hdr_len > len)
2424 			goto out_unlock;
2425 
2426 		if (vnet_hdr.gso_type != VIRTIO_NET_HDR_GSO_NONE) {
2427 			switch (vnet_hdr.gso_type & ~VIRTIO_NET_HDR_GSO_ECN) {
2428 			case VIRTIO_NET_HDR_GSO_TCPV4:
2429 				gso_type = SKB_GSO_TCPV4;
2430 				break;
2431 			case VIRTIO_NET_HDR_GSO_TCPV6:
2432 				gso_type = SKB_GSO_TCPV6;
2433 				break;
2434 			case VIRTIO_NET_HDR_GSO_UDP:
2435 				gso_type = SKB_GSO_UDP;
2436 				break;
2437 			default:
2438 				goto out_unlock;
2439 			}
2440 
2441 			if (vnet_hdr.gso_type & VIRTIO_NET_HDR_GSO_ECN)
2442 				gso_type |= SKB_GSO_TCP_ECN;
2443 
2444 			if (vnet_hdr.gso_size == 0)
2445 				goto out_unlock;
2446 
2447 		}
2448 	}
2449 
2450 	if (unlikely(sock_flag(sk, SOCK_NOFCS))) {
2451 		if (!netif_supports_nofcs(dev)) {
2452 			err = -EPROTONOSUPPORT;
2453 			goto out_unlock;
2454 		}
2455 		extra_len = 4; /* We're doing our own CRC */
2456 	}
2457 
2458 	err = -EMSGSIZE;
2459 	if (!gso_type && (len > dev->mtu + reserve + VLAN_HLEN + extra_len))
2460 		goto out_unlock;
2461 
2462 	err = -ENOBUFS;
2463 	hlen = LL_RESERVED_SPACE(dev);
2464 	tlen = dev->needed_tailroom;
2465 	skb = packet_alloc_skb(sk, hlen + tlen, hlen, len, vnet_hdr.hdr_len,
2466 			       msg->msg_flags & MSG_DONTWAIT, &err);
2467 	if (skb == NULL)
2468 		goto out_unlock;
2469 
2470 	skb_set_network_header(skb, reserve);
2471 
2472 	err = -EINVAL;
2473 	if (sock->type == SOCK_DGRAM &&
2474 	    (offset = dev_hard_header(skb, dev, ntohs(proto), addr, NULL, len)) < 0)
2475 		goto out_free;
2476 
2477 	/* Returns -EFAULT on error */
2478 	err = skb_copy_datagram_from_iovec(skb, offset, msg->msg_iov, 0, len);
2479 	if (err)
2480 		goto out_free;
2481 
2482 	sock_tx_timestamp(sk, &skb_shinfo(skb)->tx_flags);
2483 
2484 	if (!gso_type && (len > dev->mtu + reserve + extra_len)) {
2485 		/* Earlier code assumed this would be a VLAN pkt,
2486 		 * double-check this now that we have the actual
2487 		 * packet in hand.
2488 		 */
2489 		struct ethhdr *ehdr;
2490 		skb_reset_mac_header(skb);
2491 		ehdr = eth_hdr(skb);
2492 		if (ehdr->h_proto != htons(ETH_P_8021Q)) {
2493 			err = -EMSGSIZE;
2494 			goto out_free;
2495 		}
2496 	}
2497 
2498 	skb->protocol = proto;
2499 	skb->dev = dev;
2500 	skb->priority = sk->sk_priority;
2501 	skb->mark = sk->sk_mark;
2502 	skb_set_queue_mapping(skb, packet_pick_tx_queue(dev));
2503 
2504 	if (po->has_vnet_hdr) {
2505 		if (vnet_hdr.flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) {
2506 			if (!skb_partial_csum_set(skb, vnet_hdr.csum_start,
2507 						  vnet_hdr.csum_offset)) {
2508 				err = -EINVAL;
2509 				goto out_free;
2510 			}
2511 		}
2512 
2513 		skb_shinfo(skb)->gso_size = vnet_hdr.gso_size;
2514 		skb_shinfo(skb)->gso_type = gso_type;
2515 
2516 		/* Header must be checked, and gso_segs computed. */
2517 		skb_shinfo(skb)->gso_type |= SKB_GSO_DODGY;
2518 		skb_shinfo(skb)->gso_segs = 0;
2519 
2520 		len += vnet_hdr_len;
2521 	}
2522 
2523 	if (!packet_use_direct_xmit(po))
2524 		skb_probe_transport_header(skb, reserve);
2525 	if (unlikely(extra_len == 4))
2526 		skb->no_fcs = 1;
2527 
2528 	err = po->xmit(skb);
2529 	if (err > 0 && (err = net_xmit_errno(err)) != 0)
2530 		goto out_unlock;
2531 
2532 	dev_put(dev);
2533 
2534 	return len;
2535 
2536 out_free:
2537 	kfree_skb(skb);
2538 out_unlock:
2539 	if (dev)
2540 		dev_put(dev);
2541 out:
2542 	return err;
2543 }
2544 
2545 static int packet_sendmsg(struct kiocb *iocb, struct socket *sock,
2546 		struct msghdr *msg, size_t len)
2547 {
2548 	struct sock *sk = sock->sk;
2549 	struct packet_sock *po = pkt_sk(sk);
2550 
2551 	if (po->tx_ring.pg_vec)
2552 		return tpacket_snd(po, msg);
2553 	else
2554 		return packet_snd(sock, msg, len);
2555 }
2556 
2557 /*
2558  *	Close a PACKET socket. This is fairly simple. We immediately go
2559  *	to 'closed' state and remove our protocol entry in the device list.
2560  */
2561 
2562 static int packet_release(struct socket *sock)
2563 {
2564 	struct sock *sk = sock->sk;
2565 	struct packet_sock *po;
2566 	struct net *net;
2567 	union tpacket_req_u req_u;
2568 
2569 	if (!sk)
2570 		return 0;
2571 
2572 	net = sock_net(sk);
2573 	po = pkt_sk(sk);
2574 
2575 	mutex_lock(&net->packet.sklist_lock);
2576 	sk_del_node_init_rcu(sk);
2577 	mutex_unlock(&net->packet.sklist_lock);
2578 
2579 	preempt_disable();
2580 	sock_prot_inuse_add(net, sk->sk_prot, -1);
2581 	preempt_enable();
2582 
2583 	spin_lock(&po->bind_lock);
2584 	unregister_prot_hook(sk, false);
2585 	packet_cached_dev_reset(po);
2586 
2587 	if (po->prot_hook.dev) {
2588 		dev_put(po->prot_hook.dev);
2589 		po->prot_hook.dev = NULL;
2590 	}
2591 	spin_unlock(&po->bind_lock);
2592 
2593 	packet_flush_mclist(sk);
2594 
2595 	if (po->rx_ring.pg_vec) {
2596 		memset(&req_u, 0, sizeof(req_u));
2597 		packet_set_ring(sk, &req_u, 1, 0);
2598 	}
2599 
2600 	if (po->tx_ring.pg_vec) {
2601 		memset(&req_u, 0, sizeof(req_u));
2602 		packet_set_ring(sk, &req_u, 1, 1);
2603 	}
2604 
2605 	fanout_release(sk);
2606 
2607 	synchronize_net();
2608 	/*
2609 	 *	Now the socket is dead. No more input will appear.
2610 	 */
2611 	sock_orphan(sk);
2612 	sock->sk = NULL;
2613 
2614 	/* Purge queues */
2615 
2616 	skb_queue_purge(&sk->sk_receive_queue);
2617 	packet_free_pending(po);
2618 	sk_refcnt_debug_release(sk);
2619 
2620 	sock_put(sk);
2621 	return 0;
2622 }
2623 
2624 /*
2625  *	Attach a packet hook.
2626  */
2627 
2628 static int packet_do_bind(struct sock *sk, struct net_device *dev, __be16 proto)
2629 {
2630 	struct packet_sock *po = pkt_sk(sk);
2631 	const struct net_device *dev_curr;
2632 	__be16 proto_curr;
2633 	bool need_rehook;
2634 
2635 	if (po->fanout) {
2636 		if (dev)
2637 			dev_put(dev);
2638 
2639 		return -EINVAL;
2640 	}
2641 
2642 	lock_sock(sk);
2643 	spin_lock(&po->bind_lock);
2644 
2645 	proto_curr = po->prot_hook.type;
2646 	dev_curr = po->prot_hook.dev;
2647 
2648 	need_rehook = proto_curr != proto || dev_curr != dev;
2649 
2650 	if (need_rehook) {
2651 		unregister_prot_hook(sk, true);
2652 
2653 		po->num = proto;
2654 		po->prot_hook.type = proto;
2655 
2656 		if (po->prot_hook.dev)
2657 			dev_put(po->prot_hook.dev);
2658 
2659 		po->prot_hook.dev = dev;
2660 
2661 		po->ifindex = dev ? dev->ifindex : 0;
2662 		packet_cached_dev_assign(po, dev);
2663 	}
2664 
2665 	if (proto == 0 || !need_rehook)
2666 		goto out_unlock;
2667 
2668 	if (!dev || (dev->flags & IFF_UP)) {
2669 		register_prot_hook(sk);
2670 	} else {
2671 		sk->sk_err = ENETDOWN;
2672 		if (!sock_flag(sk, SOCK_DEAD))
2673 			sk->sk_error_report(sk);
2674 	}
2675 
2676 out_unlock:
2677 	spin_unlock(&po->bind_lock);
2678 	release_sock(sk);
2679 	return 0;
2680 }
2681 
2682 /*
2683  *	Bind a packet socket to a device
2684  */
2685 
2686 static int packet_bind_spkt(struct socket *sock, struct sockaddr *uaddr,
2687 			    int addr_len)
2688 {
2689 	struct sock *sk = sock->sk;
2690 	char name[15];
2691 	struct net_device *dev;
2692 	int err = -ENODEV;
2693 
2694 	/*
2695 	 *	Check legality
2696 	 */
2697 
2698 	if (addr_len != sizeof(struct sockaddr))
2699 		return -EINVAL;
2700 	strlcpy(name, uaddr->sa_data, sizeof(name));
2701 
2702 	dev = dev_get_by_name(sock_net(sk), name);
2703 	if (dev)
2704 		err = packet_do_bind(sk, dev, pkt_sk(sk)->num);
2705 	return err;
2706 }
2707 
2708 static int packet_bind(struct socket *sock, struct sockaddr *uaddr, int addr_len)
2709 {
2710 	struct sockaddr_ll *sll = (struct sockaddr_ll *)uaddr;
2711 	struct sock *sk = sock->sk;
2712 	struct net_device *dev = NULL;
2713 	int err;
2714 
2715 
2716 	/*
2717 	 *	Check legality
2718 	 */
2719 
2720 	if (addr_len < sizeof(struct sockaddr_ll))
2721 		return -EINVAL;
2722 	if (sll->sll_family != AF_PACKET)
2723 		return -EINVAL;
2724 
2725 	if (sll->sll_ifindex) {
2726 		err = -ENODEV;
2727 		dev = dev_get_by_index(sock_net(sk), sll->sll_ifindex);
2728 		if (dev == NULL)
2729 			goto out;
2730 	}
2731 	err = packet_do_bind(sk, dev, sll->sll_protocol ? : pkt_sk(sk)->num);
2732 
2733 out:
2734 	return err;
2735 }
2736 
2737 static struct proto packet_proto = {
2738 	.name	  = "PACKET",
2739 	.owner	  = THIS_MODULE,
2740 	.obj_size = sizeof(struct packet_sock),
2741 };
2742 
2743 /*
2744  *	Create a packet of type SOCK_PACKET.
2745  */
2746 
2747 static int packet_create(struct net *net, struct socket *sock, int protocol,
2748 			 int kern)
2749 {
2750 	struct sock *sk;
2751 	struct packet_sock *po;
2752 	__be16 proto = (__force __be16)protocol; /* weird, but documented */
2753 	int err;
2754 
2755 	if (!ns_capable(net->user_ns, CAP_NET_RAW))
2756 		return -EPERM;
2757 	if (sock->type != SOCK_DGRAM && sock->type != SOCK_RAW &&
2758 	    sock->type != SOCK_PACKET)
2759 		return -ESOCKTNOSUPPORT;
2760 
2761 	sock->state = SS_UNCONNECTED;
2762 
2763 	err = -ENOBUFS;
2764 	sk = sk_alloc(net, PF_PACKET, GFP_KERNEL, &packet_proto);
2765 	if (sk == NULL)
2766 		goto out;
2767 
2768 	sock->ops = &packet_ops;
2769 	if (sock->type == SOCK_PACKET)
2770 		sock->ops = &packet_ops_spkt;
2771 
2772 	sock_init_data(sock, sk);
2773 
2774 	po = pkt_sk(sk);
2775 	sk->sk_family = PF_PACKET;
2776 	po->num = proto;
2777 	po->xmit = dev_queue_xmit;
2778 
2779 	err = packet_alloc_pending(po);
2780 	if (err)
2781 		goto out2;
2782 
2783 	packet_cached_dev_reset(po);
2784 
2785 	sk->sk_destruct = packet_sock_destruct;
2786 	sk_refcnt_debug_inc(sk);
2787 
2788 	/*
2789 	 *	Attach a protocol block
2790 	 */
2791 
2792 	spin_lock_init(&po->bind_lock);
2793 	mutex_init(&po->pg_vec_lock);
2794 	po->prot_hook.func = packet_rcv;
2795 
2796 	if (sock->type == SOCK_PACKET)
2797 		po->prot_hook.func = packet_rcv_spkt;
2798 
2799 	po->prot_hook.af_packet_priv = sk;
2800 
2801 	if (proto) {
2802 		po->prot_hook.type = proto;
2803 		register_prot_hook(sk);
2804 	}
2805 
2806 	mutex_lock(&net->packet.sklist_lock);
2807 	sk_add_node_rcu(sk, &net->packet.sklist);
2808 	mutex_unlock(&net->packet.sklist_lock);
2809 
2810 	preempt_disable();
2811 	sock_prot_inuse_add(net, &packet_proto, 1);
2812 	preempt_enable();
2813 
2814 	return 0;
2815 out2:
2816 	sk_free(sk);
2817 out:
2818 	return err;
2819 }
2820 
2821 /*
2822  *	Pull a packet from our receive queue and hand it to the user.
2823  *	If necessary we block.
2824  */
2825 
2826 static int packet_recvmsg(struct kiocb *iocb, struct socket *sock,
2827 			  struct msghdr *msg, size_t len, int flags)
2828 {
2829 	struct sock *sk = sock->sk;
2830 	struct sk_buff *skb;
2831 	int copied, err;
2832 	int vnet_hdr_len = 0;
2833 
2834 	err = -EINVAL;
2835 	if (flags & ~(MSG_PEEK|MSG_DONTWAIT|MSG_TRUNC|MSG_CMSG_COMPAT|MSG_ERRQUEUE))
2836 		goto out;
2837 
2838 #if 0
2839 	/* What error should we return now? EUNATTACH? */
2840 	if (pkt_sk(sk)->ifindex < 0)
2841 		return -ENODEV;
2842 #endif
2843 
2844 	if (flags & MSG_ERRQUEUE) {
2845 		err = sock_recv_errqueue(sk, msg, len,
2846 					 SOL_PACKET, PACKET_TX_TIMESTAMP);
2847 		goto out;
2848 	}
2849 
2850 	/*
2851 	 *	Call the generic datagram receiver. This handles all sorts
2852 	 *	of horrible races and re-entrancy so we can forget about it
2853 	 *	in the protocol layers.
2854 	 *
2855 	 *	Now it will return ENETDOWN, if device have just gone down,
2856 	 *	but then it will block.
2857 	 */
2858 
2859 	skb = skb_recv_datagram(sk, flags, flags & MSG_DONTWAIT, &err);
2860 
2861 	/*
2862 	 *	An error occurred so return it. Because skb_recv_datagram()
2863 	 *	handles the blocking we don't see and worry about blocking
2864 	 *	retries.
2865 	 */
2866 
2867 	if (skb == NULL)
2868 		goto out;
2869 
2870 	if (pkt_sk(sk)->has_vnet_hdr) {
2871 		struct virtio_net_hdr vnet_hdr = { 0 };
2872 
2873 		err = -EINVAL;
2874 		vnet_hdr_len = sizeof(vnet_hdr);
2875 		if (len < vnet_hdr_len)
2876 			goto out_free;
2877 
2878 		len -= vnet_hdr_len;
2879 
2880 		if (skb_is_gso(skb)) {
2881 			struct skb_shared_info *sinfo = skb_shinfo(skb);
2882 
2883 			/* This is a hint as to how much should be linear. */
2884 			vnet_hdr.hdr_len = skb_headlen(skb);
2885 			vnet_hdr.gso_size = sinfo->gso_size;
2886 			if (sinfo->gso_type & SKB_GSO_TCPV4)
2887 				vnet_hdr.gso_type = VIRTIO_NET_HDR_GSO_TCPV4;
2888 			else if (sinfo->gso_type & SKB_GSO_TCPV6)
2889 				vnet_hdr.gso_type = VIRTIO_NET_HDR_GSO_TCPV6;
2890 			else if (sinfo->gso_type & SKB_GSO_UDP)
2891 				vnet_hdr.gso_type = VIRTIO_NET_HDR_GSO_UDP;
2892 			else if (sinfo->gso_type & SKB_GSO_FCOE)
2893 				goto out_free;
2894 			else
2895 				BUG();
2896 			if (sinfo->gso_type & SKB_GSO_TCP_ECN)
2897 				vnet_hdr.gso_type |= VIRTIO_NET_HDR_GSO_ECN;
2898 		} else
2899 			vnet_hdr.gso_type = VIRTIO_NET_HDR_GSO_NONE;
2900 
2901 		if (skb->ip_summed == CHECKSUM_PARTIAL) {
2902 			vnet_hdr.flags = VIRTIO_NET_HDR_F_NEEDS_CSUM;
2903 			vnet_hdr.csum_start = skb_checksum_start_offset(skb);
2904 			vnet_hdr.csum_offset = skb->csum_offset;
2905 		} else if (skb->ip_summed == CHECKSUM_UNNECESSARY) {
2906 			vnet_hdr.flags = VIRTIO_NET_HDR_F_DATA_VALID;
2907 		} /* else everything is zero */
2908 
2909 		err = memcpy_toiovec(msg->msg_iov, (void *)&vnet_hdr,
2910 				     vnet_hdr_len);
2911 		if (err < 0)
2912 			goto out_free;
2913 	}
2914 
2915 	/* You lose any data beyond the buffer you gave. If it worries
2916 	 * a user program they can ask the device for its MTU
2917 	 * anyway.
2918 	 */
2919 	copied = skb->len;
2920 	if (copied > len) {
2921 		copied = len;
2922 		msg->msg_flags |= MSG_TRUNC;
2923 	}
2924 
2925 	err = skb_copy_datagram_iovec(skb, 0, msg->msg_iov, copied);
2926 	if (err)
2927 		goto out_free;
2928 
2929 	sock_recv_ts_and_drops(msg, sk, skb);
2930 
2931 	if (msg->msg_name) {
2932 		/* If the address length field is there to be filled
2933 		 * in, we fill it in now.
2934 		 */
2935 		if (sock->type == SOCK_PACKET) {
2936 			__sockaddr_check_size(sizeof(struct sockaddr_pkt));
2937 			msg->msg_namelen = sizeof(struct sockaddr_pkt);
2938 		} else {
2939 			struct sockaddr_ll *sll = &PACKET_SKB_CB(skb)->sa.ll;
2940 			msg->msg_namelen = sll->sll_halen +
2941 				offsetof(struct sockaddr_ll, sll_addr);
2942 		}
2943 		memcpy(msg->msg_name, &PACKET_SKB_CB(skb)->sa,
2944 		       msg->msg_namelen);
2945 	}
2946 
2947 	if (pkt_sk(sk)->auxdata) {
2948 		struct tpacket_auxdata aux;
2949 
2950 		aux.tp_status = TP_STATUS_USER;
2951 		if (skb->ip_summed == CHECKSUM_PARTIAL)
2952 			aux.tp_status |= TP_STATUS_CSUMNOTREADY;
2953 		aux.tp_len = PACKET_SKB_CB(skb)->origlen;
2954 		aux.tp_snaplen = skb->len;
2955 		aux.tp_mac = 0;
2956 		aux.tp_net = skb_network_offset(skb);
2957 		if (vlan_tx_tag_present(skb)) {
2958 			aux.tp_vlan_tci = vlan_tx_tag_get(skb);
2959 			aux.tp_vlan_tpid = ntohs(skb->vlan_proto);
2960 			aux.tp_status |= TP_STATUS_VLAN_VALID | TP_STATUS_VLAN_TPID_VALID;
2961 		} else {
2962 			aux.tp_vlan_tci = 0;
2963 			aux.tp_vlan_tpid = 0;
2964 		}
2965 		put_cmsg(msg, SOL_PACKET, PACKET_AUXDATA, sizeof(aux), &aux);
2966 	}
2967 
2968 	/*
2969 	 *	Free or return the buffer as appropriate. Again this
2970 	 *	hides all the races and re-entrancy issues from us.
2971 	 */
2972 	err = vnet_hdr_len + ((flags&MSG_TRUNC) ? skb->len : copied);
2973 
2974 out_free:
2975 	skb_free_datagram(sk, skb);
2976 out:
2977 	return err;
2978 }
2979 
2980 static int packet_getname_spkt(struct socket *sock, struct sockaddr *uaddr,
2981 			       int *uaddr_len, int peer)
2982 {
2983 	struct net_device *dev;
2984 	struct sock *sk	= sock->sk;
2985 
2986 	if (peer)
2987 		return -EOPNOTSUPP;
2988 
2989 	uaddr->sa_family = AF_PACKET;
2990 	memset(uaddr->sa_data, 0, sizeof(uaddr->sa_data));
2991 	rcu_read_lock();
2992 	dev = dev_get_by_index_rcu(sock_net(sk), pkt_sk(sk)->ifindex);
2993 	if (dev)
2994 		strlcpy(uaddr->sa_data, dev->name, sizeof(uaddr->sa_data));
2995 	rcu_read_unlock();
2996 	*uaddr_len = sizeof(*uaddr);
2997 
2998 	return 0;
2999 }
3000 
3001 static int packet_getname(struct socket *sock, struct sockaddr *uaddr,
3002 			  int *uaddr_len, int peer)
3003 {
3004 	struct net_device *dev;
3005 	struct sock *sk = sock->sk;
3006 	struct packet_sock *po = pkt_sk(sk);
3007 	DECLARE_SOCKADDR(struct sockaddr_ll *, sll, uaddr);
3008 
3009 	if (peer)
3010 		return -EOPNOTSUPP;
3011 
3012 	sll->sll_family = AF_PACKET;
3013 	sll->sll_ifindex = po->ifindex;
3014 	sll->sll_protocol = po->num;
3015 	sll->sll_pkttype = 0;
3016 	rcu_read_lock();
3017 	dev = dev_get_by_index_rcu(sock_net(sk), po->ifindex);
3018 	if (dev) {
3019 		sll->sll_hatype = dev->type;
3020 		sll->sll_halen = dev->addr_len;
3021 		memcpy(sll->sll_addr, dev->dev_addr, dev->addr_len);
3022 	} else {
3023 		sll->sll_hatype = 0;	/* Bad: we have no ARPHRD_UNSPEC */
3024 		sll->sll_halen = 0;
3025 	}
3026 	rcu_read_unlock();
3027 	*uaddr_len = offsetof(struct sockaddr_ll, sll_addr) + sll->sll_halen;
3028 
3029 	return 0;
3030 }
3031 
3032 static int packet_dev_mc(struct net_device *dev, struct packet_mclist *i,
3033 			 int what)
3034 {
3035 	switch (i->type) {
3036 	case PACKET_MR_MULTICAST:
3037 		if (i->alen != dev->addr_len)
3038 			return -EINVAL;
3039 		if (what > 0)
3040 			return dev_mc_add(dev, i->addr);
3041 		else
3042 			return dev_mc_del(dev, i->addr);
3043 		break;
3044 	case PACKET_MR_PROMISC:
3045 		return dev_set_promiscuity(dev, what);
3046 		break;
3047 	case PACKET_MR_ALLMULTI:
3048 		return dev_set_allmulti(dev, what);
3049 		break;
3050 	case PACKET_MR_UNICAST:
3051 		if (i->alen != dev->addr_len)
3052 			return -EINVAL;
3053 		if (what > 0)
3054 			return dev_uc_add(dev, i->addr);
3055 		else
3056 			return dev_uc_del(dev, i->addr);
3057 		break;
3058 	default:
3059 		break;
3060 	}
3061 	return 0;
3062 }
3063 
3064 static void packet_dev_mclist(struct net_device *dev, struct packet_mclist *i, int what)
3065 {
3066 	for ( ; i; i = i->next) {
3067 		if (i->ifindex == dev->ifindex)
3068 			packet_dev_mc(dev, i, what);
3069 	}
3070 }
3071 
3072 static int packet_mc_add(struct sock *sk, struct packet_mreq_max *mreq)
3073 {
3074 	struct packet_sock *po = pkt_sk(sk);
3075 	struct packet_mclist *ml, *i;
3076 	struct net_device *dev;
3077 	int err;
3078 
3079 	rtnl_lock();
3080 
3081 	err = -ENODEV;
3082 	dev = __dev_get_by_index(sock_net(sk), mreq->mr_ifindex);
3083 	if (!dev)
3084 		goto done;
3085 
3086 	err = -EINVAL;
3087 	if (mreq->mr_alen > dev->addr_len)
3088 		goto done;
3089 
3090 	err = -ENOBUFS;
3091 	i = kmalloc(sizeof(*i), GFP_KERNEL);
3092 	if (i == NULL)
3093 		goto done;
3094 
3095 	err = 0;
3096 	for (ml = po->mclist; ml; ml = ml->next) {
3097 		if (ml->ifindex == mreq->mr_ifindex &&
3098 		    ml->type == mreq->mr_type &&
3099 		    ml->alen == mreq->mr_alen &&
3100 		    memcmp(ml->addr, mreq->mr_address, ml->alen) == 0) {
3101 			ml->count++;
3102 			/* Free the new element ... */
3103 			kfree(i);
3104 			goto done;
3105 		}
3106 	}
3107 
3108 	i->type = mreq->mr_type;
3109 	i->ifindex = mreq->mr_ifindex;
3110 	i->alen = mreq->mr_alen;
3111 	memcpy(i->addr, mreq->mr_address, i->alen);
3112 	i->count = 1;
3113 	i->next = po->mclist;
3114 	po->mclist = i;
3115 	err = packet_dev_mc(dev, i, 1);
3116 	if (err) {
3117 		po->mclist = i->next;
3118 		kfree(i);
3119 	}
3120 
3121 done:
3122 	rtnl_unlock();
3123 	return err;
3124 }
3125 
3126 static int packet_mc_drop(struct sock *sk, struct packet_mreq_max *mreq)
3127 {
3128 	struct packet_mclist *ml, **mlp;
3129 
3130 	rtnl_lock();
3131 
3132 	for (mlp = &pkt_sk(sk)->mclist; (ml = *mlp) != NULL; mlp = &ml->next) {
3133 		if (ml->ifindex == mreq->mr_ifindex &&
3134 		    ml->type == mreq->mr_type &&
3135 		    ml->alen == mreq->mr_alen &&
3136 		    memcmp(ml->addr, mreq->mr_address, ml->alen) == 0) {
3137 			if (--ml->count == 0) {
3138 				struct net_device *dev;
3139 				*mlp = ml->next;
3140 				dev = __dev_get_by_index(sock_net(sk), ml->ifindex);
3141 				if (dev)
3142 					packet_dev_mc(dev, ml, -1);
3143 				kfree(ml);
3144 			}
3145 			rtnl_unlock();
3146 			return 0;
3147 		}
3148 	}
3149 	rtnl_unlock();
3150 	return -EADDRNOTAVAIL;
3151 }
3152 
3153 static void packet_flush_mclist(struct sock *sk)
3154 {
3155 	struct packet_sock *po = pkt_sk(sk);
3156 	struct packet_mclist *ml;
3157 
3158 	if (!po->mclist)
3159 		return;
3160 
3161 	rtnl_lock();
3162 	while ((ml = po->mclist) != NULL) {
3163 		struct net_device *dev;
3164 
3165 		po->mclist = ml->next;
3166 		dev = __dev_get_by_index(sock_net(sk), ml->ifindex);
3167 		if (dev != NULL)
3168 			packet_dev_mc(dev, ml, -1);
3169 		kfree(ml);
3170 	}
3171 	rtnl_unlock();
3172 }
3173 
3174 static int
3175 packet_setsockopt(struct socket *sock, int level, int optname, char __user *optval, unsigned int optlen)
3176 {
3177 	struct sock *sk = sock->sk;
3178 	struct packet_sock *po = pkt_sk(sk);
3179 	int ret;
3180 
3181 	if (level != SOL_PACKET)
3182 		return -ENOPROTOOPT;
3183 
3184 	switch (optname) {
3185 	case PACKET_ADD_MEMBERSHIP:
3186 	case PACKET_DROP_MEMBERSHIP:
3187 	{
3188 		struct packet_mreq_max mreq;
3189 		int len = optlen;
3190 		memset(&mreq, 0, sizeof(mreq));
3191 		if (len < sizeof(struct packet_mreq))
3192 			return -EINVAL;
3193 		if (len > sizeof(mreq))
3194 			len = sizeof(mreq);
3195 		if (copy_from_user(&mreq, optval, len))
3196 			return -EFAULT;
3197 		if (len < (mreq.mr_alen + offsetof(struct packet_mreq, mr_address)))
3198 			return -EINVAL;
3199 		if (optname == PACKET_ADD_MEMBERSHIP)
3200 			ret = packet_mc_add(sk, &mreq);
3201 		else
3202 			ret = packet_mc_drop(sk, &mreq);
3203 		return ret;
3204 	}
3205 
3206 	case PACKET_RX_RING:
3207 	case PACKET_TX_RING:
3208 	{
3209 		union tpacket_req_u req_u;
3210 		int len;
3211 
3212 		switch (po->tp_version) {
3213 		case TPACKET_V1:
3214 		case TPACKET_V2:
3215 			len = sizeof(req_u.req);
3216 			break;
3217 		case TPACKET_V3:
3218 		default:
3219 			len = sizeof(req_u.req3);
3220 			break;
3221 		}
3222 		if (optlen < len)
3223 			return -EINVAL;
3224 		if (pkt_sk(sk)->has_vnet_hdr)
3225 			return -EINVAL;
3226 		if (copy_from_user(&req_u.req, optval, len))
3227 			return -EFAULT;
3228 		return packet_set_ring(sk, &req_u, 0,
3229 			optname == PACKET_TX_RING);
3230 	}
3231 	case PACKET_COPY_THRESH:
3232 	{
3233 		int val;
3234 
3235 		if (optlen != sizeof(val))
3236 			return -EINVAL;
3237 		if (copy_from_user(&val, optval, sizeof(val)))
3238 			return -EFAULT;
3239 
3240 		pkt_sk(sk)->copy_thresh = val;
3241 		return 0;
3242 	}
3243 	case PACKET_VERSION:
3244 	{
3245 		int val;
3246 
3247 		if (optlen != sizeof(val))
3248 			return -EINVAL;
3249 		if (po->rx_ring.pg_vec || po->tx_ring.pg_vec)
3250 			return -EBUSY;
3251 		if (copy_from_user(&val, optval, sizeof(val)))
3252 			return -EFAULT;
3253 		switch (val) {
3254 		case TPACKET_V1:
3255 		case TPACKET_V2:
3256 		case TPACKET_V3:
3257 			po->tp_version = val;
3258 			return 0;
3259 		default:
3260 			return -EINVAL;
3261 		}
3262 	}
3263 	case PACKET_RESERVE:
3264 	{
3265 		unsigned int val;
3266 
3267 		if (optlen != sizeof(val))
3268 			return -EINVAL;
3269 		if (po->rx_ring.pg_vec || po->tx_ring.pg_vec)
3270 			return -EBUSY;
3271 		if (copy_from_user(&val, optval, sizeof(val)))
3272 			return -EFAULT;
3273 		po->tp_reserve = val;
3274 		return 0;
3275 	}
3276 	case PACKET_LOSS:
3277 	{
3278 		unsigned int val;
3279 
3280 		if (optlen != sizeof(val))
3281 			return -EINVAL;
3282 		if (po->rx_ring.pg_vec || po->tx_ring.pg_vec)
3283 			return -EBUSY;
3284 		if (copy_from_user(&val, optval, sizeof(val)))
3285 			return -EFAULT;
3286 		po->tp_loss = !!val;
3287 		return 0;
3288 	}
3289 	case PACKET_AUXDATA:
3290 	{
3291 		int val;
3292 
3293 		if (optlen < sizeof(val))
3294 			return -EINVAL;
3295 		if (copy_from_user(&val, optval, sizeof(val)))
3296 			return -EFAULT;
3297 
3298 		po->auxdata = !!val;
3299 		return 0;
3300 	}
3301 	case PACKET_ORIGDEV:
3302 	{
3303 		int val;
3304 
3305 		if (optlen < sizeof(val))
3306 			return -EINVAL;
3307 		if (copy_from_user(&val, optval, sizeof(val)))
3308 			return -EFAULT;
3309 
3310 		po->origdev = !!val;
3311 		return 0;
3312 	}
3313 	case PACKET_VNET_HDR:
3314 	{
3315 		int val;
3316 
3317 		if (sock->type != SOCK_RAW)
3318 			return -EINVAL;
3319 		if (po->rx_ring.pg_vec || po->tx_ring.pg_vec)
3320 			return -EBUSY;
3321 		if (optlen < sizeof(val))
3322 			return -EINVAL;
3323 		if (copy_from_user(&val, optval, sizeof(val)))
3324 			return -EFAULT;
3325 
3326 		po->has_vnet_hdr = !!val;
3327 		return 0;
3328 	}
3329 	case PACKET_TIMESTAMP:
3330 	{
3331 		int val;
3332 
3333 		if (optlen != sizeof(val))
3334 			return -EINVAL;
3335 		if (copy_from_user(&val, optval, sizeof(val)))
3336 			return -EFAULT;
3337 
3338 		po->tp_tstamp = val;
3339 		return 0;
3340 	}
3341 	case PACKET_FANOUT:
3342 	{
3343 		int val;
3344 
3345 		if (optlen != sizeof(val))
3346 			return -EINVAL;
3347 		if (copy_from_user(&val, optval, sizeof(val)))
3348 			return -EFAULT;
3349 
3350 		return fanout_add(sk, val & 0xffff, val >> 16);
3351 	}
3352 	case PACKET_TX_HAS_OFF:
3353 	{
3354 		unsigned int val;
3355 
3356 		if (optlen != sizeof(val))
3357 			return -EINVAL;
3358 		if (po->rx_ring.pg_vec || po->tx_ring.pg_vec)
3359 			return -EBUSY;
3360 		if (copy_from_user(&val, optval, sizeof(val)))
3361 			return -EFAULT;
3362 		po->tp_tx_has_off = !!val;
3363 		return 0;
3364 	}
3365 	case PACKET_QDISC_BYPASS:
3366 	{
3367 		int val;
3368 
3369 		if (optlen != sizeof(val))
3370 			return -EINVAL;
3371 		if (copy_from_user(&val, optval, sizeof(val)))
3372 			return -EFAULT;
3373 
3374 		po->xmit = val ? packet_direct_xmit : dev_queue_xmit;
3375 		return 0;
3376 	}
3377 	default:
3378 		return -ENOPROTOOPT;
3379 	}
3380 }
3381 
3382 static int packet_getsockopt(struct socket *sock, int level, int optname,
3383 			     char __user *optval, int __user *optlen)
3384 {
3385 	int len;
3386 	int val, lv = sizeof(val);
3387 	struct sock *sk = sock->sk;
3388 	struct packet_sock *po = pkt_sk(sk);
3389 	void *data = &val;
3390 	union tpacket_stats_u st;
3391 
3392 	if (level != SOL_PACKET)
3393 		return -ENOPROTOOPT;
3394 
3395 	if (get_user(len, optlen))
3396 		return -EFAULT;
3397 
3398 	if (len < 0)
3399 		return -EINVAL;
3400 
3401 	switch (optname) {
3402 	case PACKET_STATISTICS:
3403 		spin_lock_bh(&sk->sk_receive_queue.lock);
3404 		memcpy(&st, &po->stats, sizeof(st));
3405 		memset(&po->stats, 0, sizeof(po->stats));
3406 		spin_unlock_bh(&sk->sk_receive_queue.lock);
3407 
3408 		if (po->tp_version == TPACKET_V3) {
3409 			lv = sizeof(struct tpacket_stats_v3);
3410 			st.stats3.tp_packets += st.stats3.tp_drops;
3411 			data = &st.stats3;
3412 		} else {
3413 			lv = sizeof(struct tpacket_stats);
3414 			st.stats1.tp_packets += st.stats1.tp_drops;
3415 			data = &st.stats1;
3416 		}
3417 
3418 		break;
3419 	case PACKET_AUXDATA:
3420 		val = po->auxdata;
3421 		break;
3422 	case PACKET_ORIGDEV:
3423 		val = po->origdev;
3424 		break;
3425 	case PACKET_VNET_HDR:
3426 		val = po->has_vnet_hdr;
3427 		break;
3428 	case PACKET_VERSION:
3429 		val = po->tp_version;
3430 		break;
3431 	case PACKET_HDRLEN:
3432 		if (len > sizeof(int))
3433 			len = sizeof(int);
3434 		if (copy_from_user(&val, optval, len))
3435 			return -EFAULT;
3436 		switch (val) {
3437 		case TPACKET_V1:
3438 			val = sizeof(struct tpacket_hdr);
3439 			break;
3440 		case TPACKET_V2:
3441 			val = sizeof(struct tpacket2_hdr);
3442 			break;
3443 		case TPACKET_V3:
3444 			val = sizeof(struct tpacket3_hdr);
3445 			break;
3446 		default:
3447 			return -EINVAL;
3448 		}
3449 		break;
3450 	case PACKET_RESERVE:
3451 		val = po->tp_reserve;
3452 		break;
3453 	case PACKET_LOSS:
3454 		val = po->tp_loss;
3455 		break;
3456 	case PACKET_TIMESTAMP:
3457 		val = po->tp_tstamp;
3458 		break;
3459 	case PACKET_FANOUT:
3460 		val = (po->fanout ?
3461 		       ((u32)po->fanout->id |
3462 			((u32)po->fanout->type << 16) |
3463 			((u32)po->fanout->flags << 24)) :
3464 		       0);
3465 		break;
3466 	case PACKET_TX_HAS_OFF:
3467 		val = po->tp_tx_has_off;
3468 		break;
3469 	case PACKET_QDISC_BYPASS:
3470 		val = packet_use_direct_xmit(po);
3471 		break;
3472 	default:
3473 		return -ENOPROTOOPT;
3474 	}
3475 
3476 	if (len > lv)
3477 		len = lv;
3478 	if (put_user(len, optlen))
3479 		return -EFAULT;
3480 	if (copy_to_user(optval, data, len))
3481 		return -EFAULT;
3482 	return 0;
3483 }
3484 
3485 
3486 static int packet_notifier(struct notifier_block *this,
3487 			   unsigned long msg, void *ptr)
3488 {
3489 	struct sock *sk;
3490 	struct net_device *dev = netdev_notifier_info_to_dev(ptr);
3491 	struct net *net = dev_net(dev);
3492 
3493 	rcu_read_lock();
3494 	sk_for_each_rcu(sk, &net->packet.sklist) {
3495 		struct packet_sock *po = pkt_sk(sk);
3496 
3497 		switch (msg) {
3498 		case NETDEV_UNREGISTER:
3499 			if (po->mclist)
3500 				packet_dev_mclist(dev, po->mclist, -1);
3501 			/* fallthrough */
3502 
3503 		case NETDEV_DOWN:
3504 			if (dev->ifindex == po->ifindex) {
3505 				spin_lock(&po->bind_lock);
3506 				if (po->running) {
3507 					__unregister_prot_hook(sk, false);
3508 					sk->sk_err = ENETDOWN;
3509 					if (!sock_flag(sk, SOCK_DEAD))
3510 						sk->sk_error_report(sk);
3511 				}
3512 				if (msg == NETDEV_UNREGISTER) {
3513 					packet_cached_dev_reset(po);
3514 					po->ifindex = -1;
3515 					if (po->prot_hook.dev)
3516 						dev_put(po->prot_hook.dev);
3517 					po->prot_hook.dev = NULL;
3518 				}
3519 				spin_unlock(&po->bind_lock);
3520 			}
3521 			break;
3522 		case NETDEV_UP:
3523 			if (dev->ifindex == po->ifindex) {
3524 				spin_lock(&po->bind_lock);
3525 				if (po->num)
3526 					register_prot_hook(sk);
3527 				spin_unlock(&po->bind_lock);
3528 			}
3529 			break;
3530 		}
3531 	}
3532 	rcu_read_unlock();
3533 	return NOTIFY_DONE;
3534 }
3535 
3536 
3537 static int packet_ioctl(struct socket *sock, unsigned int cmd,
3538 			unsigned long arg)
3539 {
3540 	struct sock *sk = sock->sk;
3541 
3542 	switch (cmd) {
3543 	case SIOCOUTQ:
3544 	{
3545 		int amount = sk_wmem_alloc_get(sk);
3546 
3547 		return put_user(amount, (int __user *)arg);
3548 	}
3549 	case SIOCINQ:
3550 	{
3551 		struct sk_buff *skb;
3552 		int amount = 0;
3553 
3554 		spin_lock_bh(&sk->sk_receive_queue.lock);
3555 		skb = skb_peek(&sk->sk_receive_queue);
3556 		if (skb)
3557 			amount = skb->len;
3558 		spin_unlock_bh(&sk->sk_receive_queue.lock);
3559 		return put_user(amount, (int __user *)arg);
3560 	}
3561 	case SIOCGSTAMP:
3562 		return sock_get_timestamp(sk, (struct timeval __user *)arg);
3563 	case SIOCGSTAMPNS:
3564 		return sock_get_timestampns(sk, (struct timespec __user *)arg);
3565 
3566 #ifdef CONFIG_INET
3567 	case SIOCADDRT:
3568 	case SIOCDELRT:
3569 	case SIOCDARP:
3570 	case SIOCGARP:
3571 	case SIOCSARP:
3572 	case SIOCGIFADDR:
3573 	case SIOCSIFADDR:
3574 	case SIOCGIFBRDADDR:
3575 	case SIOCSIFBRDADDR:
3576 	case SIOCGIFNETMASK:
3577 	case SIOCSIFNETMASK:
3578 	case SIOCGIFDSTADDR:
3579 	case SIOCSIFDSTADDR:
3580 	case SIOCSIFFLAGS:
3581 		return inet_dgram_ops.ioctl(sock, cmd, arg);
3582 #endif
3583 
3584 	default:
3585 		return -ENOIOCTLCMD;
3586 	}
3587 	return 0;
3588 }
3589 
3590 static unsigned int packet_poll(struct file *file, struct socket *sock,
3591 				poll_table *wait)
3592 {
3593 	struct sock *sk = sock->sk;
3594 	struct packet_sock *po = pkt_sk(sk);
3595 	unsigned int mask = datagram_poll(file, sock, wait);
3596 
3597 	spin_lock_bh(&sk->sk_receive_queue.lock);
3598 	if (po->rx_ring.pg_vec) {
3599 		if (!packet_previous_rx_frame(po, &po->rx_ring,
3600 			TP_STATUS_KERNEL))
3601 			mask |= POLLIN | POLLRDNORM;
3602 	}
3603 	spin_unlock_bh(&sk->sk_receive_queue.lock);
3604 	spin_lock_bh(&sk->sk_write_queue.lock);
3605 	if (po->tx_ring.pg_vec) {
3606 		if (packet_current_frame(po, &po->tx_ring, TP_STATUS_AVAILABLE))
3607 			mask |= POLLOUT | POLLWRNORM;
3608 	}
3609 	spin_unlock_bh(&sk->sk_write_queue.lock);
3610 	return mask;
3611 }
3612 
3613 
3614 /* Dirty? Well, I still did not learn better way to account
3615  * for user mmaps.
3616  */
3617 
3618 static void packet_mm_open(struct vm_area_struct *vma)
3619 {
3620 	struct file *file = vma->vm_file;
3621 	struct socket *sock = file->private_data;
3622 	struct sock *sk = sock->sk;
3623 
3624 	if (sk)
3625 		atomic_inc(&pkt_sk(sk)->mapped);
3626 }
3627 
3628 static void packet_mm_close(struct vm_area_struct *vma)
3629 {
3630 	struct file *file = vma->vm_file;
3631 	struct socket *sock = file->private_data;
3632 	struct sock *sk = sock->sk;
3633 
3634 	if (sk)
3635 		atomic_dec(&pkt_sk(sk)->mapped);
3636 }
3637 
3638 static const struct vm_operations_struct packet_mmap_ops = {
3639 	.open	=	packet_mm_open,
3640 	.close	=	packet_mm_close,
3641 };
3642 
3643 static void free_pg_vec(struct pgv *pg_vec, unsigned int order,
3644 			unsigned int len)
3645 {
3646 	int i;
3647 
3648 	for (i = 0; i < len; i++) {
3649 		if (likely(pg_vec[i].buffer)) {
3650 			if (is_vmalloc_addr(pg_vec[i].buffer))
3651 				vfree(pg_vec[i].buffer);
3652 			else
3653 				free_pages((unsigned long)pg_vec[i].buffer,
3654 					   order);
3655 			pg_vec[i].buffer = NULL;
3656 		}
3657 	}
3658 	kfree(pg_vec);
3659 }
3660 
3661 static char *alloc_one_pg_vec_page(unsigned long order)
3662 {
3663 	char *buffer;
3664 	gfp_t gfp_flags = GFP_KERNEL | __GFP_COMP |
3665 			  __GFP_ZERO | __GFP_NOWARN | __GFP_NORETRY;
3666 
3667 	buffer = (char *) __get_free_pages(gfp_flags, order);
3668 	if (buffer)
3669 		return buffer;
3670 
3671 	/* __get_free_pages failed, fall back to vmalloc */
3672 	buffer = vzalloc((1 << order) * PAGE_SIZE);
3673 	if (buffer)
3674 		return buffer;
3675 
3676 	/* vmalloc failed, lets dig into swap here */
3677 	gfp_flags &= ~__GFP_NORETRY;
3678 	buffer = (char *) __get_free_pages(gfp_flags, order);
3679 	if (buffer)
3680 		return buffer;
3681 
3682 	/* complete and utter failure */
3683 	return NULL;
3684 }
3685 
3686 static struct pgv *alloc_pg_vec(struct tpacket_req *req, int order)
3687 {
3688 	unsigned int block_nr = req->tp_block_nr;
3689 	struct pgv *pg_vec;
3690 	int i;
3691 
3692 	pg_vec = kcalloc(block_nr, sizeof(struct pgv), GFP_KERNEL);
3693 	if (unlikely(!pg_vec))
3694 		goto out;
3695 
3696 	for (i = 0; i < block_nr; i++) {
3697 		pg_vec[i].buffer = alloc_one_pg_vec_page(order);
3698 		if (unlikely(!pg_vec[i].buffer))
3699 			goto out_free_pgvec;
3700 	}
3701 
3702 out:
3703 	return pg_vec;
3704 
3705 out_free_pgvec:
3706 	free_pg_vec(pg_vec, order, block_nr);
3707 	pg_vec = NULL;
3708 	goto out;
3709 }
3710 
3711 static int packet_set_ring(struct sock *sk, union tpacket_req_u *req_u,
3712 		int closing, int tx_ring)
3713 {
3714 	struct pgv *pg_vec = NULL;
3715 	struct packet_sock *po = pkt_sk(sk);
3716 	int was_running, order = 0;
3717 	struct packet_ring_buffer *rb;
3718 	struct sk_buff_head *rb_queue;
3719 	__be16 num;
3720 	int err = -EINVAL;
3721 	/* Added to avoid minimal code churn */
3722 	struct tpacket_req *req = &req_u->req;
3723 
3724 	/* Opening a Tx-ring is NOT supported in TPACKET_V3 */
3725 	if (!closing && tx_ring && (po->tp_version > TPACKET_V2)) {
3726 		WARN(1, "Tx-ring is not supported.\n");
3727 		goto out;
3728 	}
3729 
3730 	rb = tx_ring ? &po->tx_ring : &po->rx_ring;
3731 	rb_queue = tx_ring ? &sk->sk_write_queue : &sk->sk_receive_queue;
3732 
3733 	err = -EBUSY;
3734 	if (!closing) {
3735 		if (atomic_read(&po->mapped))
3736 			goto out;
3737 		if (packet_read_pending(rb))
3738 			goto out;
3739 	}
3740 
3741 	if (req->tp_block_nr) {
3742 		/* Sanity tests and some calculations */
3743 		err = -EBUSY;
3744 		if (unlikely(rb->pg_vec))
3745 			goto out;
3746 
3747 		switch (po->tp_version) {
3748 		case TPACKET_V1:
3749 			po->tp_hdrlen = TPACKET_HDRLEN;
3750 			break;
3751 		case TPACKET_V2:
3752 			po->tp_hdrlen = TPACKET2_HDRLEN;
3753 			break;
3754 		case TPACKET_V3:
3755 			po->tp_hdrlen = TPACKET3_HDRLEN;
3756 			break;
3757 		}
3758 
3759 		err = -EINVAL;
3760 		if (unlikely((int)req->tp_block_size <= 0))
3761 			goto out;
3762 		if (unlikely(req->tp_block_size & (PAGE_SIZE - 1)))
3763 			goto out;
3764 		if (unlikely(req->tp_frame_size < po->tp_hdrlen +
3765 					po->tp_reserve))
3766 			goto out;
3767 		if (unlikely(req->tp_frame_size & (TPACKET_ALIGNMENT - 1)))
3768 			goto out;
3769 
3770 		rb->frames_per_block = req->tp_block_size/req->tp_frame_size;
3771 		if (unlikely(rb->frames_per_block <= 0))
3772 			goto out;
3773 		if (unlikely((rb->frames_per_block * req->tp_block_nr) !=
3774 					req->tp_frame_nr))
3775 			goto out;
3776 
3777 		err = -ENOMEM;
3778 		order = get_order(req->tp_block_size);
3779 		pg_vec = alloc_pg_vec(req, order);
3780 		if (unlikely(!pg_vec))
3781 			goto out;
3782 		switch (po->tp_version) {
3783 		case TPACKET_V3:
3784 		/* Transmit path is not supported. We checked
3785 		 * it above but just being paranoid
3786 		 */
3787 			if (!tx_ring)
3788 				init_prb_bdqc(po, rb, pg_vec, req_u, tx_ring);
3789 				break;
3790 		default:
3791 			break;
3792 		}
3793 	}
3794 	/* Done */
3795 	else {
3796 		err = -EINVAL;
3797 		if (unlikely(req->tp_frame_nr))
3798 			goto out;
3799 	}
3800 
3801 	lock_sock(sk);
3802 
3803 	/* Detach socket from network */
3804 	spin_lock(&po->bind_lock);
3805 	was_running = po->running;
3806 	num = po->num;
3807 	if (was_running) {
3808 		po->num = 0;
3809 		__unregister_prot_hook(sk, false);
3810 	}
3811 	spin_unlock(&po->bind_lock);
3812 
3813 	synchronize_net();
3814 
3815 	err = -EBUSY;
3816 	mutex_lock(&po->pg_vec_lock);
3817 	if (closing || atomic_read(&po->mapped) == 0) {
3818 		err = 0;
3819 		spin_lock_bh(&rb_queue->lock);
3820 		swap(rb->pg_vec, pg_vec);
3821 		rb->frame_max = (req->tp_frame_nr - 1);
3822 		rb->head = 0;
3823 		rb->frame_size = req->tp_frame_size;
3824 		spin_unlock_bh(&rb_queue->lock);
3825 
3826 		swap(rb->pg_vec_order, order);
3827 		swap(rb->pg_vec_len, req->tp_block_nr);
3828 
3829 		rb->pg_vec_pages = req->tp_block_size/PAGE_SIZE;
3830 		po->prot_hook.func = (po->rx_ring.pg_vec) ?
3831 						tpacket_rcv : packet_rcv;
3832 		skb_queue_purge(rb_queue);
3833 		if (atomic_read(&po->mapped))
3834 			pr_err("packet_mmap: vma is busy: %d\n",
3835 			       atomic_read(&po->mapped));
3836 	}
3837 	mutex_unlock(&po->pg_vec_lock);
3838 
3839 	spin_lock(&po->bind_lock);
3840 	if (was_running) {
3841 		po->num = num;
3842 		register_prot_hook(sk);
3843 	}
3844 	spin_unlock(&po->bind_lock);
3845 	if (closing && (po->tp_version > TPACKET_V2)) {
3846 		/* Because we don't support block-based V3 on tx-ring */
3847 		if (!tx_ring)
3848 			prb_shutdown_retire_blk_timer(po, tx_ring, rb_queue);
3849 	}
3850 	release_sock(sk);
3851 
3852 	if (pg_vec)
3853 		free_pg_vec(pg_vec, order, req->tp_block_nr);
3854 out:
3855 	return err;
3856 }
3857 
3858 static int packet_mmap(struct file *file, struct socket *sock,
3859 		struct vm_area_struct *vma)
3860 {
3861 	struct sock *sk = sock->sk;
3862 	struct packet_sock *po = pkt_sk(sk);
3863 	unsigned long size, expected_size;
3864 	struct packet_ring_buffer *rb;
3865 	unsigned long start;
3866 	int err = -EINVAL;
3867 	int i;
3868 
3869 	if (vma->vm_pgoff)
3870 		return -EINVAL;
3871 
3872 	mutex_lock(&po->pg_vec_lock);
3873 
3874 	expected_size = 0;
3875 	for (rb = &po->rx_ring; rb <= &po->tx_ring; rb++) {
3876 		if (rb->pg_vec) {
3877 			expected_size += rb->pg_vec_len
3878 						* rb->pg_vec_pages
3879 						* PAGE_SIZE;
3880 		}
3881 	}
3882 
3883 	if (expected_size == 0)
3884 		goto out;
3885 
3886 	size = vma->vm_end - vma->vm_start;
3887 	if (size != expected_size)
3888 		goto out;
3889 
3890 	start = vma->vm_start;
3891 	for (rb = &po->rx_ring; rb <= &po->tx_ring; rb++) {
3892 		if (rb->pg_vec == NULL)
3893 			continue;
3894 
3895 		for (i = 0; i < rb->pg_vec_len; i++) {
3896 			struct page *page;
3897 			void *kaddr = rb->pg_vec[i].buffer;
3898 			int pg_num;
3899 
3900 			for (pg_num = 0; pg_num < rb->pg_vec_pages; pg_num++) {
3901 				page = pgv_to_page(kaddr);
3902 				err = vm_insert_page(vma, start, page);
3903 				if (unlikely(err))
3904 					goto out;
3905 				start += PAGE_SIZE;
3906 				kaddr += PAGE_SIZE;
3907 			}
3908 		}
3909 	}
3910 
3911 	atomic_inc(&po->mapped);
3912 	vma->vm_ops = &packet_mmap_ops;
3913 	err = 0;
3914 
3915 out:
3916 	mutex_unlock(&po->pg_vec_lock);
3917 	return err;
3918 }
3919 
3920 static const struct proto_ops packet_ops_spkt = {
3921 	.family =	PF_PACKET,
3922 	.owner =	THIS_MODULE,
3923 	.release =	packet_release,
3924 	.bind =		packet_bind_spkt,
3925 	.connect =	sock_no_connect,
3926 	.socketpair =	sock_no_socketpair,
3927 	.accept =	sock_no_accept,
3928 	.getname =	packet_getname_spkt,
3929 	.poll =		datagram_poll,
3930 	.ioctl =	packet_ioctl,
3931 	.listen =	sock_no_listen,
3932 	.shutdown =	sock_no_shutdown,
3933 	.setsockopt =	sock_no_setsockopt,
3934 	.getsockopt =	sock_no_getsockopt,
3935 	.sendmsg =	packet_sendmsg_spkt,
3936 	.recvmsg =	packet_recvmsg,
3937 	.mmap =		sock_no_mmap,
3938 	.sendpage =	sock_no_sendpage,
3939 };
3940 
3941 static const struct proto_ops packet_ops = {
3942 	.family =	PF_PACKET,
3943 	.owner =	THIS_MODULE,
3944 	.release =	packet_release,
3945 	.bind =		packet_bind,
3946 	.connect =	sock_no_connect,
3947 	.socketpair =	sock_no_socketpair,
3948 	.accept =	sock_no_accept,
3949 	.getname =	packet_getname,
3950 	.poll =		packet_poll,
3951 	.ioctl =	packet_ioctl,
3952 	.listen =	sock_no_listen,
3953 	.shutdown =	sock_no_shutdown,
3954 	.setsockopt =	packet_setsockopt,
3955 	.getsockopt =	packet_getsockopt,
3956 	.sendmsg =	packet_sendmsg,
3957 	.recvmsg =	packet_recvmsg,
3958 	.mmap =		packet_mmap,
3959 	.sendpage =	sock_no_sendpage,
3960 };
3961 
3962 static const struct net_proto_family packet_family_ops = {
3963 	.family =	PF_PACKET,
3964 	.create =	packet_create,
3965 	.owner	=	THIS_MODULE,
3966 };
3967 
3968 static struct notifier_block packet_netdev_notifier = {
3969 	.notifier_call =	packet_notifier,
3970 };
3971 
3972 #ifdef CONFIG_PROC_FS
3973 
3974 static void *packet_seq_start(struct seq_file *seq, loff_t *pos)
3975 	__acquires(RCU)
3976 {
3977 	struct net *net = seq_file_net(seq);
3978 
3979 	rcu_read_lock();
3980 	return seq_hlist_start_head_rcu(&net->packet.sklist, *pos);
3981 }
3982 
3983 static void *packet_seq_next(struct seq_file *seq, void *v, loff_t *pos)
3984 {
3985 	struct net *net = seq_file_net(seq);
3986 	return seq_hlist_next_rcu(v, &net->packet.sklist, pos);
3987 }
3988 
3989 static void packet_seq_stop(struct seq_file *seq, void *v)
3990 	__releases(RCU)
3991 {
3992 	rcu_read_unlock();
3993 }
3994 
3995 static int packet_seq_show(struct seq_file *seq, void *v)
3996 {
3997 	if (v == SEQ_START_TOKEN)
3998 		seq_puts(seq, "sk       RefCnt Type Proto  Iface R Rmem   User   Inode\n");
3999 	else {
4000 		struct sock *s = sk_entry(v);
4001 		const struct packet_sock *po = pkt_sk(s);
4002 
4003 		seq_printf(seq,
4004 			   "%pK %-6d %-4d %04x   %-5d %1d %-6u %-6u %-6lu\n",
4005 			   s,
4006 			   atomic_read(&s->sk_refcnt),
4007 			   s->sk_type,
4008 			   ntohs(po->num),
4009 			   po->ifindex,
4010 			   po->running,
4011 			   atomic_read(&s->sk_rmem_alloc),
4012 			   from_kuid_munged(seq_user_ns(seq), sock_i_uid(s)),
4013 			   sock_i_ino(s));
4014 	}
4015 
4016 	return 0;
4017 }
4018 
4019 static const struct seq_operations packet_seq_ops = {
4020 	.start	= packet_seq_start,
4021 	.next	= packet_seq_next,
4022 	.stop	= packet_seq_stop,
4023 	.show	= packet_seq_show,
4024 };
4025 
4026 static int packet_seq_open(struct inode *inode, struct file *file)
4027 {
4028 	return seq_open_net(inode, file, &packet_seq_ops,
4029 			    sizeof(struct seq_net_private));
4030 }
4031 
4032 static const struct file_operations packet_seq_fops = {
4033 	.owner		= THIS_MODULE,
4034 	.open		= packet_seq_open,
4035 	.read		= seq_read,
4036 	.llseek		= seq_lseek,
4037 	.release	= seq_release_net,
4038 };
4039 
4040 #endif
4041 
4042 static int __net_init packet_net_init(struct net *net)
4043 {
4044 	mutex_init(&net->packet.sklist_lock);
4045 	INIT_HLIST_HEAD(&net->packet.sklist);
4046 
4047 	if (!proc_create("packet", 0, net->proc_net, &packet_seq_fops))
4048 		return -ENOMEM;
4049 
4050 	return 0;
4051 }
4052 
4053 static void __net_exit packet_net_exit(struct net *net)
4054 {
4055 	remove_proc_entry("packet", net->proc_net);
4056 }
4057 
4058 static struct pernet_operations packet_net_ops = {
4059 	.init = packet_net_init,
4060 	.exit = packet_net_exit,
4061 };
4062 
4063 
4064 static void __exit packet_exit(void)
4065 {
4066 	unregister_netdevice_notifier(&packet_netdev_notifier);
4067 	unregister_pernet_subsys(&packet_net_ops);
4068 	sock_unregister(PF_PACKET);
4069 	proto_unregister(&packet_proto);
4070 }
4071 
4072 static int __init packet_init(void)
4073 {
4074 	int rc = proto_register(&packet_proto, 0);
4075 
4076 	if (rc != 0)
4077 		goto out;
4078 
4079 	sock_register(&packet_family_ops);
4080 	register_pernet_subsys(&packet_net_ops);
4081 	register_netdevice_notifier(&packet_netdev_notifier);
4082 out:
4083 	return rc;
4084 }
4085 
4086 module_init(packet_init);
4087 module_exit(packet_exit);
4088 MODULE_LICENSE("GPL");
4089 MODULE_ALIAS_NETPROTO(PF_PACKET);
4090