xref: /openbmc/linux/net/packet/af_packet.c (revision 875e5771536f8f631f38f0c6090a108cd611fcf3)
1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /*
3  * INET		An implementation of the TCP/IP protocol suite for the LINUX
4  *		operating system.  INET is implemented using the  BSD Socket
5  *		interface as the means of communication with the user level.
6  *
7  *		PACKET - implements raw packet sockets.
8  *
9  * Authors:	Ross Biro
10  *		Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
11  *		Alan Cox, <gw4pts@gw4pts.ampr.org>
12  *
13  * Fixes:
14  *		Alan Cox	:	verify_area() now used correctly
15  *		Alan Cox	:	new skbuff lists, look ma no backlogs!
16  *		Alan Cox	:	tidied skbuff lists.
17  *		Alan Cox	:	Now uses generic datagram routines I
18  *					added. Also fixed the peek/read crash
19  *					from all old Linux datagram code.
20  *		Alan Cox	:	Uses the improved datagram code.
21  *		Alan Cox	:	Added NULL's for socket options.
22  *		Alan Cox	:	Re-commented the code.
23  *		Alan Cox	:	Use new kernel side addressing
24  *		Rob Janssen	:	Correct MTU usage.
25  *		Dave Platt	:	Counter leaks caused by incorrect
26  *					interrupt locking and some slightly
27  *					dubious gcc output. Can you read
28  *					compiler: it said _VOLATILE_
29  *	Richard Kooijman	:	Timestamp fixes.
30  *		Alan Cox	:	New buffers. Use sk->mac.raw.
31  *		Alan Cox	:	sendmsg/recvmsg support.
32  *		Alan Cox	:	Protocol setting support
33  *	Alexey Kuznetsov	:	Untied from IPv4 stack.
34  *	Cyrus Durgin		:	Fixed kerneld for kmod.
35  *	Michal Ostrowski        :       Module initialization cleanup.
36  *         Ulises Alonso        :       Frame number limit removal and
37  *                                      packet_set_ring memory leak.
38  *		Eric Biederman	:	Allow for > 8 byte hardware addresses.
39  *					The convention is that longer addresses
40  *					will simply extend the hardware address
41  *					byte arrays at the end of sockaddr_ll
42  *					and packet_mreq.
43  *		Johann Baudy	:	Added TX RING.
44  *		Chetan Loke	:	Implemented TPACKET_V3 block abstraction
45  *					layer.
46  *					Copyright (C) 2011, <lokec@ccs.neu.edu>
47  */
48 
49 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
50 
51 #include <linux/ethtool.h>
52 #include <linux/filter.h>
53 #include <linux/types.h>
54 #include <linux/mm.h>
55 #include <linux/capability.h>
56 #include <linux/fcntl.h>
57 #include <linux/socket.h>
58 #include <linux/in.h>
59 #include <linux/inet.h>
60 #include <linux/netdevice.h>
61 #include <linux/if_packet.h>
62 #include <linux/wireless.h>
63 #include <linux/kernel.h>
64 #include <linux/kmod.h>
65 #include <linux/slab.h>
66 #include <linux/vmalloc.h>
67 #include <net/net_namespace.h>
68 #include <net/ip.h>
69 #include <net/protocol.h>
70 #include <linux/skbuff.h>
71 #include <net/sock.h>
72 #include <linux/errno.h>
73 #include <linux/timer.h>
74 #include <linux/uaccess.h>
75 #include <asm/ioctls.h>
76 #include <asm/page.h>
77 #include <asm/cacheflush.h>
78 #include <asm/io.h>
79 #include <linux/proc_fs.h>
80 #include <linux/seq_file.h>
81 #include <linux/poll.h>
82 #include <linux/module.h>
83 #include <linux/init.h>
84 #include <linux/mutex.h>
85 #include <linux/if_vlan.h>
86 #include <linux/virtio_net.h>
87 #include <linux/errqueue.h>
88 #include <linux/net_tstamp.h>
89 #include <linux/percpu.h>
90 #ifdef CONFIG_INET
91 #include <net/inet_common.h>
92 #endif
93 #include <linux/bpf.h>
94 #include <net/compat.h>
95 #include <linux/netfilter_netdev.h>
96 
97 #include "internal.h"
98 
99 /*
100    Assumptions:
101    - If the device has no dev->header_ops->create, there is no LL header
102      visible above the device. In this case, its hard_header_len should be 0.
103      The device may prepend its own header internally. In this case, its
104      needed_headroom should be set to the space needed for it to add its
105      internal header.
106      For example, a WiFi driver pretending to be an Ethernet driver should
107      set its hard_header_len to be the Ethernet header length, and set its
108      needed_headroom to be (the real WiFi header length - the fake Ethernet
109      header length).
110    - packet socket receives packets with pulled ll header,
111      so that SOCK_RAW should push it back.
112 
113 On receive:
114 -----------
115 
116 Incoming, dev_has_header(dev) == true
117    mac_header -> ll header
118    data       -> data
119 
120 Outgoing, dev_has_header(dev) == true
121    mac_header -> ll header
122    data       -> ll header
123 
124 Incoming, dev_has_header(dev) == false
125    mac_header -> data
126      However drivers often make it point to the ll header.
127      This is incorrect because the ll header should be invisible to us.
128    data       -> data
129 
130 Outgoing, dev_has_header(dev) == false
131    mac_header -> data. ll header is invisible to us.
132    data       -> data
133 
134 Resume
135   If dev_has_header(dev) == false we are unable to restore the ll header,
136     because it is invisible to us.
137 
138 
139 On transmit:
140 ------------
141 
142 dev_has_header(dev) == true
143    mac_header -> ll header
144    data       -> ll header
145 
146 dev_has_header(dev) == false (ll header is invisible to us)
147    mac_header -> data
148    data       -> data
149 
150    We should set network_header on output to the correct position,
151    packet classifier depends on it.
152  */
153 
154 /* Private packet socket structures. */
155 
156 /* identical to struct packet_mreq except it has
157  * a longer address field.
158  */
159 struct packet_mreq_max {
160 	int		mr_ifindex;
161 	unsigned short	mr_type;
162 	unsigned short	mr_alen;
163 	unsigned char	mr_address[MAX_ADDR_LEN];
164 };
165 
166 union tpacket_uhdr {
167 	struct tpacket_hdr  *h1;
168 	struct tpacket2_hdr *h2;
169 	struct tpacket3_hdr *h3;
170 	void *raw;
171 };
172 
173 static int packet_set_ring(struct sock *sk, union tpacket_req_u *req_u,
174 		int closing, int tx_ring);
175 
176 #define V3_ALIGNMENT	(8)
177 
178 #define BLK_HDR_LEN	(ALIGN(sizeof(struct tpacket_block_desc), V3_ALIGNMENT))
179 
180 #define BLK_PLUS_PRIV(sz_of_priv) \
181 	(BLK_HDR_LEN + ALIGN((sz_of_priv), V3_ALIGNMENT))
182 
183 #define BLOCK_STATUS(x)	((x)->hdr.bh1.block_status)
184 #define BLOCK_NUM_PKTS(x)	((x)->hdr.bh1.num_pkts)
185 #define BLOCK_O2FP(x)		((x)->hdr.bh1.offset_to_first_pkt)
186 #define BLOCK_LEN(x)		((x)->hdr.bh1.blk_len)
187 #define BLOCK_SNUM(x)		((x)->hdr.bh1.seq_num)
188 #define BLOCK_O2PRIV(x)	((x)->offset_to_priv)
189 
190 struct packet_sock;
191 static int tpacket_rcv(struct sk_buff *skb, struct net_device *dev,
192 		       struct packet_type *pt, struct net_device *orig_dev);
193 
194 static void *packet_previous_frame(struct packet_sock *po,
195 		struct packet_ring_buffer *rb,
196 		int status);
197 static void packet_increment_head(struct packet_ring_buffer *buff);
198 static int prb_curr_blk_in_use(struct tpacket_block_desc *);
199 static void *prb_dispatch_next_block(struct tpacket_kbdq_core *,
200 			struct packet_sock *);
201 static void prb_retire_current_block(struct tpacket_kbdq_core *,
202 		struct packet_sock *, unsigned int status);
203 static int prb_queue_frozen(struct tpacket_kbdq_core *);
204 static void prb_open_block(struct tpacket_kbdq_core *,
205 		struct tpacket_block_desc *);
206 static void prb_retire_rx_blk_timer_expired(struct timer_list *);
207 static void _prb_refresh_rx_retire_blk_timer(struct tpacket_kbdq_core *);
208 static void prb_fill_rxhash(struct tpacket_kbdq_core *, struct tpacket3_hdr *);
209 static void prb_clear_rxhash(struct tpacket_kbdq_core *,
210 		struct tpacket3_hdr *);
211 static void prb_fill_vlan_info(struct tpacket_kbdq_core *,
212 		struct tpacket3_hdr *);
213 static void packet_flush_mclist(struct sock *sk);
214 static u16 packet_pick_tx_queue(struct sk_buff *skb);
215 
216 struct packet_skb_cb {
217 	union {
218 		struct sockaddr_pkt pkt;
219 		union {
220 			/* Trick: alias skb original length with
221 			 * ll.sll_family and ll.protocol in order
222 			 * to save room.
223 			 */
224 			unsigned int origlen;
225 			struct sockaddr_ll ll;
226 		};
227 	} sa;
228 };
229 
230 #define vio_le() virtio_legacy_is_little_endian()
231 
232 #define PACKET_SKB_CB(__skb)	((struct packet_skb_cb *)((__skb)->cb))
233 
234 #define GET_PBDQC_FROM_RB(x)	((struct tpacket_kbdq_core *)(&(x)->prb_bdqc))
235 #define GET_PBLOCK_DESC(x, bid)	\
236 	((struct tpacket_block_desc *)((x)->pkbdq[(bid)].buffer))
237 #define GET_CURR_PBLOCK_DESC_FROM_CORE(x)	\
238 	((struct tpacket_block_desc *)((x)->pkbdq[(x)->kactive_blk_num].buffer))
239 #define GET_NEXT_PRB_BLK_NUM(x) \
240 	(((x)->kactive_blk_num < ((x)->knum_blocks-1)) ? \
241 	((x)->kactive_blk_num+1) : 0)
242 
243 static void __fanout_unlink(struct sock *sk, struct packet_sock *po);
244 static void __fanout_link(struct sock *sk, struct packet_sock *po);
245 
246 #ifdef CONFIG_NETFILTER_EGRESS
nf_hook_direct_egress(struct sk_buff * skb)247 static noinline struct sk_buff *nf_hook_direct_egress(struct sk_buff *skb)
248 {
249 	struct sk_buff *next, *head = NULL, *tail;
250 	int rc;
251 
252 	rcu_read_lock();
253 	for (; skb != NULL; skb = next) {
254 		next = skb->next;
255 		skb_mark_not_on_list(skb);
256 
257 		if (!nf_hook_egress(skb, &rc, skb->dev))
258 			continue;
259 
260 		if (!head)
261 			head = skb;
262 		else
263 			tail->next = skb;
264 
265 		tail = skb;
266 	}
267 	rcu_read_unlock();
268 
269 	return head;
270 }
271 #endif
272 
packet_xmit(const struct packet_sock * po,struct sk_buff * skb)273 static int packet_xmit(const struct packet_sock *po, struct sk_buff *skb)
274 {
275 	if (!packet_sock_flag(po, PACKET_SOCK_QDISC_BYPASS))
276 		return dev_queue_xmit(skb);
277 
278 #ifdef CONFIG_NETFILTER_EGRESS
279 	if (nf_hook_egress_active()) {
280 		skb = nf_hook_direct_egress(skb);
281 		if (!skb)
282 			return NET_XMIT_DROP;
283 	}
284 #endif
285 	return dev_direct_xmit(skb, packet_pick_tx_queue(skb));
286 }
287 
packet_cached_dev_get(struct packet_sock * po)288 static struct net_device *packet_cached_dev_get(struct packet_sock *po)
289 {
290 	struct net_device *dev;
291 
292 	rcu_read_lock();
293 	dev = rcu_dereference(po->cached_dev);
294 	dev_hold(dev);
295 	rcu_read_unlock();
296 
297 	return dev;
298 }
299 
packet_cached_dev_assign(struct packet_sock * po,struct net_device * dev)300 static void packet_cached_dev_assign(struct packet_sock *po,
301 				     struct net_device *dev)
302 {
303 	rcu_assign_pointer(po->cached_dev, dev);
304 }
305 
packet_cached_dev_reset(struct packet_sock * po)306 static void packet_cached_dev_reset(struct packet_sock *po)
307 {
308 	RCU_INIT_POINTER(po->cached_dev, NULL);
309 }
310 
packet_pick_tx_queue(struct sk_buff * skb)311 static u16 packet_pick_tx_queue(struct sk_buff *skb)
312 {
313 	struct net_device *dev = skb->dev;
314 	const struct net_device_ops *ops = dev->netdev_ops;
315 	int cpu = raw_smp_processor_id();
316 	u16 queue_index;
317 
318 #ifdef CONFIG_XPS
319 	skb->sender_cpu = cpu + 1;
320 #endif
321 	skb_record_rx_queue(skb, cpu % dev->real_num_tx_queues);
322 	if (ops->ndo_select_queue) {
323 		queue_index = ops->ndo_select_queue(dev, skb, NULL);
324 		queue_index = netdev_cap_txqueue(dev, queue_index);
325 	} else {
326 		queue_index = netdev_pick_tx(dev, skb, NULL);
327 	}
328 
329 	return queue_index;
330 }
331 
332 /* __register_prot_hook must be invoked through register_prot_hook
333  * or from a context in which asynchronous accesses to the packet
334  * socket is not possible (packet_create()).
335  */
__register_prot_hook(struct sock * sk)336 static void __register_prot_hook(struct sock *sk)
337 {
338 	struct packet_sock *po = pkt_sk(sk);
339 
340 	if (!packet_sock_flag(po, PACKET_SOCK_RUNNING)) {
341 		if (po->fanout)
342 			__fanout_link(sk, po);
343 		else
344 			dev_add_pack(&po->prot_hook);
345 
346 		sock_hold(sk);
347 		packet_sock_flag_set(po, PACKET_SOCK_RUNNING, 1);
348 	}
349 }
350 
register_prot_hook(struct sock * sk)351 static void register_prot_hook(struct sock *sk)
352 {
353 	lockdep_assert_held_once(&pkt_sk(sk)->bind_lock);
354 	__register_prot_hook(sk);
355 }
356 
357 /* If the sync parameter is true, we will temporarily drop
358  * the po->bind_lock and do a synchronize_net to make sure no
359  * asynchronous packet processing paths still refer to the elements
360  * of po->prot_hook.  If the sync parameter is false, it is the
361  * callers responsibility to take care of this.
362  */
__unregister_prot_hook(struct sock * sk,bool sync)363 static void __unregister_prot_hook(struct sock *sk, bool sync)
364 {
365 	struct packet_sock *po = pkt_sk(sk);
366 
367 	lockdep_assert_held_once(&po->bind_lock);
368 
369 	packet_sock_flag_set(po, PACKET_SOCK_RUNNING, 0);
370 
371 	if (po->fanout)
372 		__fanout_unlink(sk, po);
373 	else
374 		__dev_remove_pack(&po->prot_hook);
375 
376 	__sock_put(sk);
377 
378 	if (sync) {
379 		spin_unlock(&po->bind_lock);
380 		synchronize_net();
381 		spin_lock(&po->bind_lock);
382 	}
383 }
384 
unregister_prot_hook(struct sock * sk,bool sync)385 static void unregister_prot_hook(struct sock *sk, bool sync)
386 {
387 	struct packet_sock *po = pkt_sk(sk);
388 
389 	if (packet_sock_flag(po, PACKET_SOCK_RUNNING))
390 		__unregister_prot_hook(sk, sync);
391 }
392 
pgv_to_page(void * addr)393 static inline struct page * __pure pgv_to_page(void *addr)
394 {
395 	if (is_vmalloc_addr(addr))
396 		return vmalloc_to_page(addr);
397 	return virt_to_page(addr);
398 }
399 
__packet_set_status(struct packet_sock * po,void * frame,int status)400 static void __packet_set_status(struct packet_sock *po, void *frame, int status)
401 {
402 	union tpacket_uhdr h;
403 
404 	/* WRITE_ONCE() are paired with READ_ONCE() in __packet_get_status */
405 
406 	h.raw = frame;
407 	switch (po->tp_version) {
408 	case TPACKET_V1:
409 		WRITE_ONCE(h.h1->tp_status, status);
410 		flush_dcache_page(pgv_to_page(&h.h1->tp_status));
411 		break;
412 	case TPACKET_V2:
413 		WRITE_ONCE(h.h2->tp_status, status);
414 		flush_dcache_page(pgv_to_page(&h.h2->tp_status));
415 		break;
416 	case TPACKET_V3:
417 		WRITE_ONCE(h.h3->tp_status, status);
418 		flush_dcache_page(pgv_to_page(&h.h3->tp_status));
419 		break;
420 	default:
421 		WARN(1, "TPACKET version not supported.\n");
422 		BUG();
423 	}
424 
425 	smp_wmb();
426 }
427 
__packet_get_status(const struct packet_sock * po,void * frame)428 static int __packet_get_status(const struct packet_sock *po, void *frame)
429 {
430 	union tpacket_uhdr h;
431 
432 	smp_rmb();
433 
434 	/* READ_ONCE() are paired with WRITE_ONCE() in __packet_set_status */
435 
436 	h.raw = frame;
437 	switch (po->tp_version) {
438 	case TPACKET_V1:
439 		flush_dcache_page(pgv_to_page(&h.h1->tp_status));
440 		return READ_ONCE(h.h1->tp_status);
441 	case TPACKET_V2:
442 		flush_dcache_page(pgv_to_page(&h.h2->tp_status));
443 		return READ_ONCE(h.h2->tp_status);
444 	case TPACKET_V3:
445 		flush_dcache_page(pgv_to_page(&h.h3->tp_status));
446 		return READ_ONCE(h.h3->tp_status);
447 	default:
448 		WARN(1, "TPACKET version not supported.\n");
449 		BUG();
450 		return 0;
451 	}
452 }
453 
tpacket_get_timestamp(struct sk_buff * skb,struct timespec64 * ts,unsigned int flags)454 static __u32 tpacket_get_timestamp(struct sk_buff *skb, struct timespec64 *ts,
455 				   unsigned int flags)
456 {
457 	struct skb_shared_hwtstamps *shhwtstamps = skb_hwtstamps(skb);
458 
459 	if (shhwtstamps &&
460 	    (flags & SOF_TIMESTAMPING_RAW_HARDWARE) &&
461 	    ktime_to_timespec64_cond(shhwtstamps->hwtstamp, ts))
462 		return TP_STATUS_TS_RAW_HARDWARE;
463 
464 	if ((flags & SOF_TIMESTAMPING_SOFTWARE) &&
465 	    ktime_to_timespec64_cond(skb_tstamp(skb), ts))
466 		return TP_STATUS_TS_SOFTWARE;
467 
468 	return 0;
469 }
470 
__packet_set_timestamp(struct packet_sock * po,void * frame,struct sk_buff * skb)471 static __u32 __packet_set_timestamp(struct packet_sock *po, void *frame,
472 				    struct sk_buff *skb)
473 {
474 	union tpacket_uhdr h;
475 	struct timespec64 ts;
476 	__u32 ts_status;
477 
478 	if (!(ts_status = tpacket_get_timestamp(skb, &ts, READ_ONCE(po->tp_tstamp))))
479 		return 0;
480 
481 	h.raw = frame;
482 	/*
483 	 * versions 1 through 3 overflow the timestamps in y2106, since they
484 	 * all store the seconds in a 32-bit unsigned integer.
485 	 * If we create a version 4, that should have a 64-bit timestamp,
486 	 * either 64-bit seconds + 32-bit nanoseconds, or just 64-bit
487 	 * nanoseconds.
488 	 */
489 	switch (po->tp_version) {
490 	case TPACKET_V1:
491 		h.h1->tp_sec = ts.tv_sec;
492 		h.h1->tp_usec = ts.tv_nsec / NSEC_PER_USEC;
493 		break;
494 	case TPACKET_V2:
495 		h.h2->tp_sec = ts.tv_sec;
496 		h.h2->tp_nsec = ts.tv_nsec;
497 		break;
498 	case TPACKET_V3:
499 		h.h3->tp_sec = ts.tv_sec;
500 		h.h3->tp_nsec = ts.tv_nsec;
501 		break;
502 	default:
503 		WARN(1, "TPACKET version not supported.\n");
504 		BUG();
505 	}
506 
507 	/* one flush is safe, as both fields always lie on the same cacheline */
508 	flush_dcache_page(pgv_to_page(&h.h1->tp_sec));
509 	smp_wmb();
510 
511 	return ts_status;
512 }
513 
packet_lookup_frame(const struct packet_sock * po,const struct packet_ring_buffer * rb,unsigned int position,int status)514 static void *packet_lookup_frame(const struct packet_sock *po,
515 				 const struct packet_ring_buffer *rb,
516 				 unsigned int position,
517 				 int status)
518 {
519 	unsigned int pg_vec_pos, frame_offset;
520 	union tpacket_uhdr h;
521 
522 	pg_vec_pos = position / rb->frames_per_block;
523 	frame_offset = position % rb->frames_per_block;
524 
525 	h.raw = rb->pg_vec[pg_vec_pos].buffer +
526 		(frame_offset * rb->frame_size);
527 
528 	if (status != __packet_get_status(po, h.raw))
529 		return NULL;
530 
531 	return h.raw;
532 }
533 
packet_current_frame(struct packet_sock * po,struct packet_ring_buffer * rb,int status)534 static void *packet_current_frame(struct packet_sock *po,
535 		struct packet_ring_buffer *rb,
536 		int status)
537 {
538 	return packet_lookup_frame(po, rb, rb->head, status);
539 }
540 
vlan_get_tci(const struct sk_buff * skb,struct net_device * dev)541 static u16 vlan_get_tci(const struct sk_buff *skb, struct net_device *dev)
542 {
543 	struct vlan_hdr vhdr, *vh;
544 	unsigned int header_len;
545 
546 	if (!dev)
547 		return 0;
548 
549 	/* In the SOCK_DGRAM scenario, skb data starts at the network
550 	 * protocol, which is after the VLAN headers. The outer VLAN
551 	 * header is at the hard_header_len offset in non-variable
552 	 * length link layer headers. If it's a VLAN device, the
553 	 * min_header_len should be used to exclude the VLAN header
554 	 * size.
555 	 */
556 	if (dev->min_header_len == dev->hard_header_len)
557 		header_len = dev->hard_header_len;
558 	else if (is_vlan_dev(dev))
559 		header_len = dev->min_header_len;
560 	else
561 		return 0;
562 
563 	vh = skb_header_pointer(skb, skb_mac_offset(skb) + header_len,
564 				sizeof(vhdr), &vhdr);
565 	if (unlikely(!vh))
566 		return 0;
567 
568 	return ntohs(vh->h_vlan_TCI);
569 }
570 
vlan_get_protocol_dgram(const struct sk_buff * skb)571 static __be16 vlan_get_protocol_dgram(const struct sk_buff *skb)
572 {
573 	__be16 proto = skb->protocol;
574 
575 	if (unlikely(eth_type_vlan(proto)))
576 		proto = __vlan_get_protocol_offset(skb, proto,
577 						   skb_mac_offset(skb), NULL);
578 
579 	return proto;
580 }
581 
prb_del_retire_blk_timer(struct tpacket_kbdq_core * pkc)582 static void prb_del_retire_blk_timer(struct tpacket_kbdq_core *pkc)
583 {
584 	del_timer_sync(&pkc->retire_blk_timer);
585 }
586 
prb_shutdown_retire_blk_timer(struct packet_sock * po,struct sk_buff_head * rb_queue)587 static void prb_shutdown_retire_blk_timer(struct packet_sock *po,
588 		struct sk_buff_head *rb_queue)
589 {
590 	struct tpacket_kbdq_core *pkc;
591 
592 	pkc = GET_PBDQC_FROM_RB(&po->rx_ring);
593 
594 	spin_lock_bh(&rb_queue->lock);
595 	pkc->delete_blk_timer = 1;
596 	spin_unlock_bh(&rb_queue->lock);
597 
598 	prb_del_retire_blk_timer(pkc);
599 }
600 
prb_setup_retire_blk_timer(struct packet_sock * po)601 static void prb_setup_retire_blk_timer(struct packet_sock *po)
602 {
603 	struct tpacket_kbdq_core *pkc;
604 
605 	pkc = GET_PBDQC_FROM_RB(&po->rx_ring);
606 	timer_setup(&pkc->retire_blk_timer, prb_retire_rx_blk_timer_expired,
607 		    0);
608 	pkc->retire_blk_timer.expires = jiffies;
609 }
610 
prb_calc_retire_blk_tmo(struct packet_sock * po,int blk_size_in_bytes)611 static int prb_calc_retire_blk_tmo(struct packet_sock *po,
612 				int blk_size_in_bytes)
613 {
614 	struct net_device *dev;
615 	unsigned int mbits, div;
616 	struct ethtool_link_ksettings ecmd;
617 	int err;
618 
619 	rtnl_lock();
620 	dev = __dev_get_by_index(sock_net(&po->sk), po->ifindex);
621 	if (unlikely(!dev)) {
622 		rtnl_unlock();
623 		return DEFAULT_PRB_RETIRE_TOV;
624 	}
625 	err = __ethtool_get_link_ksettings(dev, &ecmd);
626 	rtnl_unlock();
627 	if (err)
628 		return DEFAULT_PRB_RETIRE_TOV;
629 
630 	/* If the link speed is so slow you don't really
631 	 * need to worry about perf anyways
632 	 */
633 	if (ecmd.base.speed < SPEED_1000 ||
634 	    ecmd.base.speed == SPEED_UNKNOWN)
635 		return DEFAULT_PRB_RETIRE_TOV;
636 
637 	div = ecmd.base.speed / 1000;
638 	mbits = (blk_size_in_bytes * 8) / (1024 * 1024);
639 
640 	if (div)
641 		mbits /= div;
642 
643 	if (div)
644 		return mbits + 1;
645 	return mbits;
646 }
647 
prb_init_ft_ops(struct tpacket_kbdq_core * p1,union tpacket_req_u * req_u)648 static void prb_init_ft_ops(struct tpacket_kbdq_core *p1,
649 			union tpacket_req_u *req_u)
650 {
651 	p1->feature_req_word = req_u->req3.tp_feature_req_word;
652 }
653 
init_prb_bdqc(struct packet_sock * po,struct packet_ring_buffer * rb,struct pgv * pg_vec,union tpacket_req_u * req_u)654 static void init_prb_bdqc(struct packet_sock *po,
655 			struct packet_ring_buffer *rb,
656 			struct pgv *pg_vec,
657 			union tpacket_req_u *req_u)
658 {
659 	struct tpacket_kbdq_core *p1 = GET_PBDQC_FROM_RB(rb);
660 	struct tpacket_block_desc *pbd;
661 
662 	memset(p1, 0x0, sizeof(*p1));
663 
664 	p1->knxt_seq_num = 1;
665 	p1->pkbdq = pg_vec;
666 	pbd = (struct tpacket_block_desc *)pg_vec[0].buffer;
667 	p1->pkblk_start	= pg_vec[0].buffer;
668 	p1->kblk_size = req_u->req3.tp_block_size;
669 	p1->knum_blocks	= req_u->req3.tp_block_nr;
670 	p1->hdrlen = po->tp_hdrlen;
671 	p1->version = po->tp_version;
672 	p1->last_kactive_blk_num = 0;
673 	po->stats.stats3.tp_freeze_q_cnt = 0;
674 	if (req_u->req3.tp_retire_blk_tov)
675 		p1->retire_blk_tov = req_u->req3.tp_retire_blk_tov;
676 	else
677 		p1->retire_blk_tov = prb_calc_retire_blk_tmo(po,
678 						req_u->req3.tp_block_size);
679 	p1->tov_in_jiffies = msecs_to_jiffies(p1->retire_blk_tov);
680 	p1->blk_sizeof_priv = req_u->req3.tp_sizeof_priv;
681 	rwlock_init(&p1->blk_fill_in_prog_lock);
682 
683 	p1->max_frame_len = p1->kblk_size - BLK_PLUS_PRIV(p1->blk_sizeof_priv);
684 	prb_init_ft_ops(p1, req_u);
685 	prb_setup_retire_blk_timer(po);
686 	prb_open_block(p1, pbd);
687 }
688 
689 /*  Do NOT update the last_blk_num first.
690  *  Assumes sk_buff_head lock is held.
691  */
_prb_refresh_rx_retire_blk_timer(struct tpacket_kbdq_core * pkc)692 static void _prb_refresh_rx_retire_blk_timer(struct tpacket_kbdq_core *pkc)
693 {
694 	mod_timer(&pkc->retire_blk_timer,
695 			jiffies + pkc->tov_in_jiffies);
696 	pkc->last_kactive_blk_num = pkc->kactive_blk_num;
697 }
698 
699 /*
700  * Timer logic:
701  * 1) We refresh the timer only when we open a block.
702  *    By doing this we don't waste cycles refreshing the timer
703  *	  on packet-by-packet basis.
704  *
705  * With a 1MB block-size, on a 1Gbps line, it will take
706  * i) ~8 ms to fill a block + ii) memcpy etc.
707  * In this cut we are not accounting for the memcpy time.
708  *
709  * So, if the user sets the 'tmo' to 10ms then the timer
710  * will never fire while the block is still getting filled
711  * (which is what we want). However, the user could choose
712  * to close a block early and that's fine.
713  *
714  * But when the timer does fire, we check whether or not to refresh it.
715  * Since the tmo granularity is in msecs, it is not too expensive
716  * to refresh the timer, lets say every '8' msecs.
717  * Either the user can set the 'tmo' or we can derive it based on
718  * a) line-speed and b) block-size.
719  * prb_calc_retire_blk_tmo() calculates the tmo.
720  *
721  */
prb_retire_rx_blk_timer_expired(struct timer_list * t)722 static void prb_retire_rx_blk_timer_expired(struct timer_list *t)
723 {
724 	struct packet_sock *po =
725 		from_timer(po, t, rx_ring.prb_bdqc.retire_blk_timer);
726 	struct tpacket_kbdq_core *pkc = GET_PBDQC_FROM_RB(&po->rx_ring);
727 	unsigned int frozen;
728 	struct tpacket_block_desc *pbd;
729 
730 	spin_lock(&po->sk.sk_receive_queue.lock);
731 
732 	frozen = prb_queue_frozen(pkc);
733 	pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
734 
735 	if (unlikely(pkc->delete_blk_timer))
736 		goto out;
737 
738 	/* We only need to plug the race when the block is partially filled.
739 	 * tpacket_rcv:
740 	 *		lock(); increment BLOCK_NUM_PKTS; unlock()
741 	 *		copy_bits() is in progress ...
742 	 *		timer fires on other cpu:
743 	 *		we can't retire the current block because copy_bits
744 	 *		is in progress.
745 	 *
746 	 */
747 	if (BLOCK_NUM_PKTS(pbd)) {
748 		/* Waiting for skb_copy_bits to finish... */
749 		write_lock(&pkc->blk_fill_in_prog_lock);
750 		write_unlock(&pkc->blk_fill_in_prog_lock);
751 	}
752 
753 	if (pkc->last_kactive_blk_num == pkc->kactive_blk_num) {
754 		if (!frozen) {
755 			if (!BLOCK_NUM_PKTS(pbd)) {
756 				/* An empty block. Just refresh the timer. */
757 				goto refresh_timer;
758 			}
759 			prb_retire_current_block(pkc, po, TP_STATUS_BLK_TMO);
760 			if (!prb_dispatch_next_block(pkc, po))
761 				goto refresh_timer;
762 			else
763 				goto out;
764 		} else {
765 			/* Case 1. Queue was frozen because user-space was
766 			 *	   lagging behind.
767 			 */
768 			if (prb_curr_blk_in_use(pbd)) {
769 				/*
770 				 * Ok, user-space is still behind.
771 				 * So just refresh the timer.
772 				 */
773 				goto refresh_timer;
774 			} else {
775 			       /* Case 2. queue was frozen,user-space caught up,
776 				* now the link went idle && the timer fired.
777 				* We don't have a block to close.So we open this
778 				* block and restart the timer.
779 				* opening a block thaws the queue,restarts timer
780 				* Thawing/timer-refresh is a side effect.
781 				*/
782 				prb_open_block(pkc, pbd);
783 				goto out;
784 			}
785 		}
786 	}
787 
788 refresh_timer:
789 	_prb_refresh_rx_retire_blk_timer(pkc);
790 
791 out:
792 	spin_unlock(&po->sk.sk_receive_queue.lock);
793 }
794 
prb_flush_block(struct tpacket_kbdq_core * pkc1,struct tpacket_block_desc * pbd1,__u32 status)795 static void prb_flush_block(struct tpacket_kbdq_core *pkc1,
796 		struct tpacket_block_desc *pbd1, __u32 status)
797 {
798 	/* Flush everything minus the block header */
799 
800 #if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE == 1
801 	u8 *start, *end;
802 
803 	start = (u8 *)pbd1;
804 
805 	/* Skip the block header(we know header WILL fit in 4K) */
806 	start += PAGE_SIZE;
807 
808 	end = (u8 *)PAGE_ALIGN((unsigned long)pkc1->pkblk_end);
809 	for (; start < end; start += PAGE_SIZE)
810 		flush_dcache_page(pgv_to_page(start));
811 
812 	smp_wmb();
813 #endif
814 
815 	/* Now update the block status. */
816 
817 	BLOCK_STATUS(pbd1) = status;
818 
819 	/* Flush the block header */
820 
821 #if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE == 1
822 	start = (u8 *)pbd1;
823 	flush_dcache_page(pgv_to_page(start));
824 
825 	smp_wmb();
826 #endif
827 }
828 
829 /*
830  * Side effect:
831  *
832  * 1) flush the block
833  * 2) Increment active_blk_num
834  *
835  * Note:We DONT refresh the timer on purpose.
836  *	Because almost always the next block will be opened.
837  */
prb_close_block(struct tpacket_kbdq_core * pkc1,struct tpacket_block_desc * pbd1,struct packet_sock * po,unsigned int stat)838 static void prb_close_block(struct tpacket_kbdq_core *pkc1,
839 		struct tpacket_block_desc *pbd1,
840 		struct packet_sock *po, unsigned int stat)
841 {
842 	__u32 status = TP_STATUS_USER | stat;
843 
844 	struct tpacket3_hdr *last_pkt;
845 	struct tpacket_hdr_v1 *h1 = &pbd1->hdr.bh1;
846 	struct sock *sk = &po->sk;
847 
848 	if (atomic_read(&po->tp_drops))
849 		status |= TP_STATUS_LOSING;
850 
851 	last_pkt = (struct tpacket3_hdr *)pkc1->prev;
852 	last_pkt->tp_next_offset = 0;
853 
854 	/* Get the ts of the last pkt */
855 	if (BLOCK_NUM_PKTS(pbd1)) {
856 		h1->ts_last_pkt.ts_sec = last_pkt->tp_sec;
857 		h1->ts_last_pkt.ts_nsec	= last_pkt->tp_nsec;
858 	} else {
859 		/* Ok, we tmo'd - so get the current time.
860 		 *
861 		 * It shouldn't really happen as we don't close empty
862 		 * blocks. See prb_retire_rx_blk_timer_expired().
863 		 */
864 		struct timespec64 ts;
865 		ktime_get_real_ts64(&ts);
866 		h1->ts_last_pkt.ts_sec = ts.tv_sec;
867 		h1->ts_last_pkt.ts_nsec	= ts.tv_nsec;
868 	}
869 
870 	smp_wmb();
871 
872 	/* Flush the block */
873 	prb_flush_block(pkc1, pbd1, status);
874 
875 	sk->sk_data_ready(sk);
876 
877 	pkc1->kactive_blk_num = GET_NEXT_PRB_BLK_NUM(pkc1);
878 }
879 
prb_thaw_queue(struct tpacket_kbdq_core * pkc)880 static void prb_thaw_queue(struct tpacket_kbdq_core *pkc)
881 {
882 	pkc->reset_pending_on_curr_blk = 0;
883 }
884 
885 /*
886  * Side effect of opening a block:
887  *
888  * 1) prb_queue is thawed.
889  * 2) retire_blk_timer is refreshed.
890  *
891  */
prb_open_block(struct tpacket_kbdq_core * pkc1,struct tpacket_block_desc * pbd1)892 static void prb_open_block(struct tpacket_kbdq_core *pkc1,
893 	struct tpacket_block_desc *pbd1)
894 {
895 	struct timespec64 ts;
896 	struct tpacket_hdr_v1 *h1 = &pbd1->hdr.bh1;
897 
898 	smp_rmb();
899 
900 	/* We could have just memset this but we will lose the
901 	 * flexibility of making the priv area sticky
902 	 */
903 
904 	BLOCK_SNUM(pbd1) = pkc1->knxt_seq_num++;
905 	BLOCK_NUM_PKTS(pbd1) = 0;
906 	BLOCK_LEN(pbd1) = BLK_PLUS_PRIV(pkc1->blk_sizeof_priv);
907 
908 	ktime_get_real_ts64(&ts);
909 
910 	h1->ts_first_pkt.ts_sec = ts.tv_sec;
911 	h1->ts_first_pkt.ts_nsec = ts.tv_nsec;
912 
913 	pkc1->pkblk_start = (char *)pbd1;
914 	pkc1->nxt_offset = pkc1->pkblk_start + BLK_PLUS_PRIV(pkc1->blk_sizeof_priv);
915 
916 	BLOCK_O2FP(pbd1) = (__u32)BLK_PLUS_PRIV(pkc1->blk_sizeof_priv);
917 	BLOCK_O2PRIV(pbd1) = BLK_HDR_LEN;
918 
919 	pbd1->version = pkc1->version;
920 	pkc1->prev = pkc1->nxt_offset;
921 	pkc1->pkblk_end = pkc1->pkblk_start + pkc1->kblk_size;
922 
923 	prb_thaw_queue(pkc1);
924 	_prb_refresh_rx_retire_blk_timer(pkc1);
925 
926 	smp_wmb();
927 }
928 
929 /*
930  * Queue freeze logic:
931  * 1) Assume tp_block_nr = 8 blocks.
932  * 2) At time 't0', user opens Rx ring.
933  * 3) Some time past 't0', kernel starts filling blocks starting from 0 .. 7
934  * 4) user-space is either sleeping or processing block '0'.
935  * 5) tpacket_rcv is currently filling block '7', since there is no space left,
936  *    it will close block-7,loop around and try to fill block '0'.
937  *    call-flow:
938  *    __packet_lookup_frame_in_block
939  *      prb_retire_current_block()
940  *      prb_dispatch_next_block()
941  *        |->(BLOCK_STATUS == USER) evaluates to true
942  *    5.1) Since block-0 is currently in-use, we just freeze the queue.
943  * 6) Now there are two cases:
944  *    6.1) Link goes idle right after the queue is frozen.
945  *         But remember, the last open_block() refreshed the timer.
946  *         When this timer expires,it will refresh itself so that we can
947  *         re-open block-0 in near future.
948  *    6.2) Link is busy and keeps on receiving packets. This is a simple
949  *         case and __packet_lookup_frame_in_block will check if block-0
950  *         is free and can now be re-used.
951  */
prb_freeze_queue(struct tpacket_kbdq_core * pkc,struct packet_sock * po)952 static void prb_freeze_queue(struct tpacket_kbdq_core *pkc,
953 				  struct packet_sock *po)
954 {
955 	pkc->reset_pending_on_curr_blk = 1;
956 	po->stats.stats3.tp_freeze_q_cnt++;
957 }
958 
959 #define TOTAL_PKT_LEN_INCL_ALIGN(length) (ALIGN((length), V3_ALIGNMENT))
960 
961 /*
962  * If the next block is free then we will dispatch it
963  * and return a good offset.
964  * Else, we will freeze the queue.
965  * So, caller must check the return value.
966  */
prb_dispatch_next_block(struct tpacket_kbdq_core * pkc,struct packet_sock * po)967 static void *prb_dispatch_next_block(struct tpacket_kbdq_core *pkc,
968 		struct packet_sock *po)
969 {
970 	struct tpacket_block_desc *pbd;
971 
972 	smp_rmb();
973 
974 	/* 1. Get current block num */
975 	pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
976 
977 	/* 2. If this block is currently in_use then freeze the queue */
978 	if (TP_STATUS_USER & BLOCK_STATUS(pbd)) {
979 		prb_freeze_queue(pkc, po);
980 		return NULL;
981 	}
982 
983 	/*
984 	 * 3.
985 	 * open this block and return the offset where the first packet
986 	 * needs to get stored.
987 	 */
988 	prb_open_block(pkc, pbd);
989 	return (void *)pkc->nxt_offset;
990 }
991 
prb_retire_current_block(struct tpacket_kbdq_core * pkc,struct packet_sock * po,unsigned int status)992 static void prb_retire_current_block(struct tpacket_kbdq_core *pkc,
993 		struct packet_sock *po, unsigned int status)
994 {
995 	struct tpacket_block_desc *pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
996 
997 	/* retire/close the current block */
998 	if (likely(TP_STATUS_KERNEL == BLOCK_STATUS(pbd))) {
999 		/*
1000 		 * Plug the case where copy_bits() is in progress on
1001 		 * cpu-0 and tpacket_rcv() got invoked on cpu-1, didn't
1002 		 * have space to copy the pkt in the current block and
1003 		 * called prb_retire_current_block()
1004 		 *
1005 		 * We don't need to worry about the TMO case because
1006 		 * the timer-handler already handled this case.
1007 		 */
1008 		if (!(status & TP_STATUS_BLK_TMO)) {
1009 			/* Waiting for skb_copy_bits to finish... */
1010 			write_lock(&pkc->blk_fill_in_prog_lock);
1011 			write_unlock(&pkc->blk_fill_in_prog_lock);
1012 		}
1013 		prb_close_block(pkc, pbd, po, status);
1014 		return;
1015 	}
1016 }
1017 
prb_curr_blk_in_use(struct tpacket_block_desc * pbd)1018 static int prb_curr_blk_in_use(struct tpacket_block_desc *pbd)
1019 {
1020 	return TP_STATUS_USER & BLOCK_STATUS(pbd);
1021 }
1022 
prb_queue_frozen(struct tpacket_kbdq_core * pkc)1023 static int prb_queue_frozen(struct tpacket_kbdq_core *pkc)
1024 {
1025 	return pkc->reset_pending_on_curr_blk;
1026 }
1027 
prb_clear_blk_fill_status(struct packet_ring_buffer * rb)1028 static void prb_clear_blk_fill_status(struct packet_ring_buffer *rb)
1029 	__releases(&pkc->blk_fill_in_prog_lock)
1030 {
1031 	struct tpacket_kbdq_core *pkc  = GET_PBDQC_FROM_RB(rb);
1032 
1033 	read_unlock(&pkc->blk_fill_in_prog_lock);
1034 }
1035 
prb_fill_rxhash(struct tpacket_kbdq_core * pkc,struct tpacket3_hdr * ppd)1036 static void prb_fill_rxhash(struct tpacket_kbdq_core *pkc,
1037 			struct tpacket3_hdr *ppd)
1038 {
1039 	ppd->hv1.tp_rxhash = skb_get_hash(pkc->skb);
1040 }
1041 
prb_clear_rxhash(struct tpacket_kbdq_core * pkc,struct tpacket3_hdr * ppd)1042 static void prb_clear_rxhash(struct tpacket_kbdq_core *pkc,
1043 			struct tpacket3_hdr *ppd)
1044 {
1045 	ppd->hv1.tp_rxhash = 0;
1046 }
1047 
prb_fill_vlan_info(struct tpacket_kbdq_core * pkc,struct tpacket3_hdr * ppd)1048 static void prb_fill_vlan_info(struct tpacket_kbdq_core *pkc,
1049 			struct tpacket3_hdr *ppd)
1050 {
1051 	struct packet_sock *po = container_of(pkc, struct packet_sock, rx_ring.prb_bdqc);
1052 
1053 	if (skb_vlan_tag_present(pkc->skb)) {
1054 		ppd->hv1.tp_vlan_tci = skb_vlan_tag_get(pkc->skb);
1055 		ppd->hv1.tp_vlan_tpid = ntohs(pkc->skb->vlan_proto);
1056 		ppd->tp_status = TP_STATUS_VLAN_VALID | TP_STATUS_VLAN_TPID_VALID;
1057 	} else if (unlikely(po->sk.sk_type == SOCK_DGRAM && eth_type_vlan(pkc->skb->protocol))) {
1058 		ppd->hv1.tp_vlan_tci = vlan_get_tci(pkc->skb, pkc->skb->dev);
1059 		ppd->hv1.tp_vlan_tpid = ntohs(pkc->skb->protocol);
1060 		ppd->tp_status = TP_STATUS_VLAN_VALID | TP_STATUS_VLAN_TPID_VALID;
1061 	} else {
1062 		ppd->hv1.tp_vlan_tci = 0;
1063 		ppd->hv1.tp_vlan_tpid = 0;
1064 		ppd->tp_status = TP_STATUS_AVAILABLE;
1065 	}
1066 }
1067 
prb_run_all_ft_ops(struct tpacket_kbdq_core * pkc,struct tpacket3_hdr * ppd)1068 static void prb_run_all_ft_ops(struct tpacket_kbdq_core *pkc,
1069 			struct tpacket3_hdr *ppd)
1070 {
1071 	ppd->hv1.tp_padding = 0;
1072 	prb_fill_vlan_info(pkc, ppd);
1073 
1074 	if (pkc->feature_req_word & TP_FT_REQ_FILL_RXHASH)
1075 		prb_fill_rxhash(pkc, ppd);
1076 	else
1077 		prb_clear_rxhash(pkc, ppd);
1078 }
1079 
prb_fill_curr_block(char * curr,struct tpacket_kbdq_core * pkc,struct tpacket_block_desc * pbd,unsigned int len)1080 static void prb_fill_curr_block(char *curr,
1081 				struct tpacket_kbdq_core *pkc,
1082 				struct tpacket_block_desc *pbd,
1083 				unsigned int len)
1084 	__acquires(&pkc->blk_fill_in_prog_lock)
1085 {
1086 	struct tpacket3_hdr *ppd;
1087 
1088 	ppd  = (struct tpacket3_hdr *)curr;
1089 	ppd->tp_next_offset = TOTAL_PKT_LEN_INCL_ALIGN(len);
1090 	pkc->prev = curr;
1091 	pkc->nxt_offset += TOTAL_PKT_LEN_INCL_ALIGN(len);
1092 	BLOCK_LEN(pbd) += TOTAL_PKT_LEN_INCL_ALIGN(len);
1093 	BLOCK_NUM_PKTS(pbd) += 1;
1094 	read_lock(&pkc->blk_fill_in_prog_lock);
1095 	prb_run_all_ft_ops(pkc, ppd);
1096 }
1097 
1098 /* Assumes caller has the sk->rx_queue.lock */
__packet_lookup_frame_in_block(struct packet_sock * po,struct sk_buff * skb,unsigned int len)1099 static void *__packet_lookup_frame_in_block(struct packet_sock *po,
1100 					    struct sk_buff *skb,
1101 					    unsigned int len
1102 					    )
1103 {
1104 	struct tpacket_kbdq_core *pkc;
1105 	struct tpacket_block_desc *pbd;
1106 	char *curr, *end;
1107 
1108 	pkc = GET_PBDQC_FROM_RB(&po->rx_ring);
1109 	pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
1110 
1111 	/* Queue is frozen when user space is lagging behind */
1112 	if (prb_queue_frozen(pkc)) {
1113 		/*
1114 		 * Check if that last block which caused the queue to freeze,
1115 		 * is still in_use by user-space.
1116 		 */
1117 		if (prb_curr_blk_in_use(pbd)) {
1118 			/* Can't record this packet */
1119 			return NULL;
1120 		} else {
1121 			/*
1122 			 * Ok, the block was released by user-space.
1123 			 * Now let's open that block.
1124 			 * opening a block also thaws the queue.
1125 			 * Thawing is a side effect.
1126 			 */
1127 			prb_open_block(pkc, pbd);
1128 		}
1129 	}
1130 
1131 	smp_mb();
1132 	curr = pkc->nxt_offset;
1133 	pkc->skb = skb;
1134 	end = (char *)pbd + pkc->kblk_size;
1135 
1136 	/* first try the current block */
1137 	if (curr+TOTAL_PKT_LEN_INCL_ALIGN(len) < end) {
1138 		prb_fill_curr_block(curr, pkc, pbd, len);
1139 		return (void *)curr;
1140 	}
1141 
1142 	/* Ok, close the current block */
1143 	prb_retire_current_block(pkc, po, 0);
1144 
1145 	/* Now, try to dispatch the next block */
1146 	curr = (char *)prb_dispatch_next_block(pkc, po);
1147 	if (curr) {
1148 		pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
1149 		prb_fill_curr_block(curr, pkc, pbd, len);
1150 		return (void *)curr;
1151 	}
1152 
1153 	/*
1154 	 * No free blocks are available.user_space hasn't caught up yet.
1155 	 * Queue was just frozen and now this packet will get dropped.
1156 	 */
1157 	return NULL;
1158 }
1159 
packet_current_rx_frame(struct packet_sock * po,struct sk_buff * skb,int status,unsigned int len)1160 static void *packet_current_rx_frame(struct packet_sock *po,
1161 					    struct sk_buff *skb,
1162 					    int status, unsigned int len)
1163 {
1164 	char *curr = NULL;
1165 	switch (po->tp_version) {
1166 	case TPACKET_V1:
1167 	case TPACKET_V2:
1168 		curr = packet_lookup_frame(po, &po->rx_ring,
1169 					po->rx_ring.head, status);
1170 		return curr;
1171 	case TPACKET_V3:
1172 		return __packet_lookup_frame_in_block(po, skb, len);
1173 	default:
1174 		WARN(1, "TPACKET version not supported\n");
1175 		BUG();
1176 		return NULL;
1177 	}
1178 }
1179 
prb_lookup_block(const struct packet_sock * po,const struct packet_ring_buffer * rb,unsigned int idx,int status)1180 static void *prb_lookup_block(const struct packet_sock *po,
1181 			      const struct packet_ring_buffer *rb,
1182 			      unsigned int idx,
1183 			      int status)
1184 {
1185 	struct tpacket_kbdq_core *pkc  = GET_PBDQC_FROM_RB(rb);
1186 	struct tpacket_block_desc *pbd = GET_PBLOCK_DESC(pkc, idx);
1187 
1188 	if (status != BLOCK_STATUS(pbd))
1189 		return NULL;
1190 	return pbd;
1191 }
1192 
prb_previous_blk_num(struct packet_ring_buffer * rb)1193 static int prb_previous_blk_num(struct packet_ring_buffer *rb)
1194 {
1195 	unsigned int prev;
1196 	if (rb->prb_bdqc.kactive_blk_num)
1197 		prev = rb->prb_bdqc.kactive_blk_num-1;
1198 	else
1199 		prev = rb->prb_bdqc.knum_blocks-1;
1200 	return prev;
1201 }
1202 
1203 /* Assumes caller has held the rx_queue.lock */
__prb_previous_block(struct packet_sock * po,struct packet_ring_buffer * rb,int status)1204 static void *__prb_previous_block(struct packet_sock *po,
1205 					 struct packet_ring_buffer *rb,
1206 					 int status)
1207 {
1208 	unsigned int previous = prb_previous_blk_num(rb);
1209 	return prb_lookup_block(po, rb, previous, status);
1210 }
1211 
packet_previous_rx_frame(struct packet_sock * po,struct packet_ring_buffer * rb,int status)1212 static void *packet_previous_rx_frame(struct packet_sock *po,
1213 					     struct packet_ring_buffer *rb,
1214 					     int status)
1215 {
1216 	if (po->tp_version <= TPACKET_V2)
1217 		return packet_previous_frame(po, rb, status);
1218 
1219 	return __prb_previous_block(po, rb, status);
1220 }
1221 
packet_increment_rx_head(struct packet_sock * po,struct packet_ring_buffer * rb)1222 static void packet_increment_rx_head(struct packet_sock *po,
1223 					    struct packet_ring_buffer *rb)
1224 {
1225 	switch (po->tp_version) {
1226 	case TPACKET_V1:
1227 	case TPACKET_V2:
1228 		return packet_increment_head(rb);
1229 	case TPACKET_V3:
1230 	default:
1231 		WARN(1, "TPACKET version not supported.\n");
1232 		BUG();
1233 		return;
1234 	}
1235 }
1236 
packet_previous_frame(struct packet_sock * po,struct packet_ring_buffer * rb,int status)1237 static void *packet_previous_frame(struct packet_sock *po,
1238 		struct packet_ring_buffer *rb,
1239 		int status)
1240 {
1241 	unsigned int previous = rb->head ? rb->head - 1 : rb->frame_max;
1242 	return packet_lookup_frame(po, rb, previous, status);
1243 }
1244 
packet_increment_head(struct packet_ring_buffer * buff)1245 static void packet_increment_head(struct packet_ring_buffer *buff)
1246 {
1247 	buff->head = buff->head != buff->frame_max ? buff->head+1 : 0;
1248 }
1249 
packet_inc_pending(struct packet_ring_buffer * rb)1250 static void packet_inc_pending(struct packet_ring_buffer *rb)
1251 {
1252 	this_cpu_inc(*rb->pending_refcnt);
1253 }
1254 
packet_dec_pending(struct packet_ring_buffer * rb)1255 static void packet_dec_pending(struct packet_ring_buffer *rb)
1256 {
1257 	this_cpu_dec(*rb->pending_refcnt);
1258 }
1259 
packet_read_pending(const struct packet_ring_buffer * rb)1260 static unsigned int packet_read_pending(const struct packet_ring_buffer *rb)
1261 {
1262 	unsigned int refcnt = 0;
1263 	int cpu;
1264 
1265 	/* We don't use pending refcount in rx_ring. */
1266 	if (rb->pending_refcnt == NULL)
1267 		return 0;
1268 
1269 	for_each_possible_cpu(cpu)
1270 		refcnt += *per_cpu_ptr(rb->pending_refcnt, cpu);
1271 
1272 	return refcnt;
1273 }
1274 
packet_alloc_pending(struct packet_sock * po)1275 static int packet_alloc_pending(struct packet_sock *po)
1276 {
1277 	po->rx_ring.pending_refcnt = NULL;
1278 
1279 	po->tx_ring.pending_refcnt = alloc_percpu(unsigned int);
1280 	if (unlikely(po->tx_ring.pending_refcnt == NULL))
1281 		return -ENOBUFS;
1282 
1283 	return 0;
1284 }
1285 
packet_free_pending(struct packet_sock * po)1286 static void packet_free_pending(struct packet_sock *po)
1287 {
1288 	free_percpu(po->tx_ring.pending_refcnt);
1289 }
1290 
1291 #define ROOM_POW_OFF	2
1292 #define ROOM_NONE	0x0
1293 #define ROOM_LOW	0x1
1294 #define ROOM_NORMAL	0x2
1295 
__tpacket_has_room(const struct packet_sock * po,int pow_off)1296 static bool __tpacket_has_room(const struct packet_sock *po, int pow_off)
1297 {
1298 	int idx, len;
1299 
1300 	len = READ_ONCE(po->rx_ring.frame_max) + 1;
1301 	idx = READ_ONCE(po->rx_ring.head);
1302 	if (pow_off)
1303 		idx += len >> pow_off;
1304 	if (idx >= len)
1305 		idx -= len;
1306 	return packet_lookup_frame(po, &po->rx_ring, idx, TP_STATUS_KERNEL);
1307 }
1308 
__tpacket_v3_has_room(const struct packet_sock * po,int pow_off)1309 static bool __tpacket_v3_has_room(const struct packet_sock *po, int pow_off)
1310 {
1311 	int idx, len;
1312 
1313 	len = READ_ONCE(po->rx_ring.prb_bdqc.knum_blocks);
1314 	idx = READ_ONCE(po->rx_ring.prb_bdqc.kactive_blk_num);
1315 	if (pow_off)
1316 		idx += len >> pow_off;
1317 	if (idx >= len)
1318 		idx -= len;
1319 	return prb_lookup_block(po, &po->rx_ring, idx, TP_STATUS_KERNEL);
1320 }
1321 
__packet_rcv_has_room(const struct packet_sock * po,const struct sk_buff * skb)1322 static int __packet_rcv_has_room(const struct packet_sock *po,
1323 				 const struct sk_buff *skb)
1324 {
1325 	const struct sock *sk = &po->sk;
1326 	int ret = ROOM_NONE;
1327 
1328 	if (po->prot_hook.func != tpacket_rcv) {
1329 		int rcvbuf = READ_ONCE(sk->sk_rcvbuf);
1330 		int avail = rcvbuf - atomic_read(&sk->sk_rmem_alloc)
1331 				   - (skb ? skb->truesize : 0);
1332 
1333 		if (avail > (rcvbuf >> ROOM_POW_OFF))
1334 			return ROOM_NORMAL;
1335 		else if (avail > 0)
1336 			return ROOM_LOW;
1337 		else
1338 			return ROOM_NONE;
1339 	}
1340 
1341 	if (po->tp_version == TPACKET_V3) {
1342 		if (__tpacket_v3_has_room(po, ROOM_POW_OFF))
1343 			ret = ROOM_NORMAL;
1344 		else if (__tpacket_v3_has_room(po, 0))
1345 			ret = ROOM_LOW;
1346 	} else {
1347 		if (__tpacket_has_room(po, ROOM_POW_OFF))
1348 			ret = ROOM_NORMAL;
1349 		else if (__tpacket_has_room(po, 0))
1350 			ret = ROOM_LOW;
1351 	}
1352 
1353 	return ret;
1354 }
1355 
packet_rcv_has_room(struct packet_sock * po,struct sk_buff * skb)1356 static int packet_rcv_has_room(struct packet_sock *po, struct sk_buff *skb)
1357 {
1358 	bool pressure;
1359 	int ret;
1360 
1361 	ret = __packet_rcv_has_room(po, skb);
1362 	pressure = ret != ROOM_NORMAL;
1363 
1364 	if (packet_sock_flag(po, PACKET_SOCK_PRESSURE) != pressure)
1365 		packet_sock_flag_set(po, PACKET_SOCK_PRESSURE, pressure);
1366 
1367 	return ret;
1368 }
1369 
packet_rcv_try_clear_pressure(struct packet_sock * po)1370 static void packet_rcv_try_clear_pressure(struct packet_sock *po)
1371 {
1372 	if (packet_sock_flag(po, PACKET_SOCK_PRESSURE) &&
1373 	    __packet_rcv_has_room(po, NULL) == ROOM_NORMAL)
1374 		packet_sock_flag_set(po, PACKET_SOCK_PRESSURE, false);
1375 }
1376 
packet_sock_destruct(struct sock * sk)1377 static void packet_sock_destruct(struct sock *sk)
1378 {
1379 	skb_queue_purge(&sk->sk_error_queue);
1380 
1381 	WARN_ON(atomic_read(&sk->sk_rmem_alloc));
1382 	WARN_ON(refcount_read(&sk->sk_wmem_alloc));
1383 
1384 	if (!sock_flag(sk, SOCK_DEAD)) {
1385 		pr_err("Attempt to release alive packet socket: %p\n", sk);
1386 		return;
1387 	}
1388 }
1389 
fanout_flow_is_huge(struct packet_sock * po,struct sk_buff * skb)1390 static bool fanout_flow_is_huge(struct packet_sock *po, struct sk_buff *skb)
1391 {
1392 	u32 *history = po->rollover->history;
1393 	u32 victim, rxhash;
1394 	int i, count = 0;
1395 
1396 	rxhash = skb_get_hash(skb);
1397 	for (i = 0; i < ROLLOVER_HLEN; i++)
1398 		if (READ_ONCE(history[i]) == rxhash)
1399 			count++;
1400 
1401 	victim = get_random_u32_below(ROLLOVER_HLEN);
1402 
1403 	/* Avoid dirtying the cache line if possible */
1404 	if (READ_ONCE(history[victim]) != rxhash)
1405 		WRITE_ONCE(history[victim], rxhash);
1406 
1407 	return count > (ROLLOVER_HLEN >> 1);
1408 }
1409 
fanout_demux_hash(struct packet_fanout * f,struct sk_buff * skb,unsigned int num)1410 static unsigned int fanout_demux_hash(struct packet_fanout *f,
1411 				      struct sk_buff *skb,
1412 				      unsigned int num)
1413 {
1414 	return reciprocal_scale(__skb_get_hash_symmetric(skb), num);
1415 }
1416 
fanout_demux_lb(struct packet_fanout * f,struct sk_buff * skb,unsigned int num)1417 static unsigned int fanout_demux_lb(struct packet_fanout *f,
1418 				    struct sk_buff *skb,
1419 				    unsigned int num)
1420 {
1421 	unsigned int val = atomic_inc_return(&f->rr_cur);
1422 
1423 	return val % num;
1424 }
1425 
fanout_demux_cpu(struct packet_fanout * f,struct sk_buff * skb,unsigned int num)1426 static unsigned int fanout_demux_cpu(struct packet_fanout *f,
1427 				     struct sk_buff *skb,
1428 				     unsigned int num)
1429 {
1430 	return smp_processor_id() % num;
1431 }
1432 
fanout_demux_rnd(struct packet_fanout * f,struct sk_buff * skb,unsigned int num)1433 static unsigned int fanout_demux_rnd(struct packet_fanout *f,
1434 				     struct sk_buff *skb,
1435 				     unsigned int num)
1436 {
1437 	return get_random_u32_below(num);
1438 }
1439 
fanout_demux_rollover(struct packet_fanout * f,struct sk_buff * skb,unsigned int idx,bool try_self,unsigned int num)1440 static unsigned int fanout_demux_rollover(struct packet_fanout *f,
1441 					  struct sk_buff *skb,
1442 					  unsigned int idx, bool try_self,
1443 					  unsigned int num)
1444 {
1445 	struct packet_sock *po, *po_next, *po_skip = NULL;
1446 	unsigned int i, j, room = ROOM_NONE;
1447 
1448 	po = pkt_sk(rcu_dereference(f->arr[idx]));
1449 
1450 	if (try_self) {
1451 		room = packet_rcv_has_room(po, skb);
1452 		if (room == ROOM_NORMAL ||
1453 		    (room == ROOM_LOW && !fanout_flow_is_huge(po, skb)))
1454 			return idx;
1455 		po_skip = po;
1456 	}
1457 
1458 	i = j = min_t(int, po->rollover->sock, num - 1);
1459 	do {
1460 		po_next = pkt_sk(rcu_dereference(f->arr[i]));
1461 		if (po_next != po_skip &&
1462 		    !packet_sock_flag(po_next, PACKET_SOCK_PRESSURE) &&
1463 		    packet_rcv_has_room(po_next, skb) == ROOM_NORMAL) {
1464 			if (i != j)
1465 				po->rollover->sock = i;
1466 			atomic_long_inc(&po->rollover->num);
1467 			if (room == ROOM_LOW)
1468 				atomic_long_inc(&po->rollover->num_huge);
1469 			return i;
1470 		}
1471 
1472 		if (++i == num)
1473 			i = 0;
1474 	} while (i != j);
1475 
1476 	atomic_long_inc(&po->rollover->num_failed);
1477 	return idx;
1478 }
1479 
fanout_demux_qm(struct packet_fanout * f,struct sk_buff * skb,unsigned int num)1480 static unsigned int fanout_demux_qm(struct packet_fanout *f,
1481 				    struct sk_buff *skb,
1482 				    unsigned int num)
1483 {
1484 	return skb_get_queue_mapping(skb) % num;
1485 }
1486 
fanout_demux_bpf(struct packet_fanout * f,struct sk_buff * skb,unsigned int num)1487 static unsigned int fanout_demux_bpf(struct packet_fanout *f,
1488 				     struct sk_buff *skb,
1489 				     unsigned int num)
1490 {
1491 	struct bpf_prog *prog;
1492 	unsigned int ret = 0;
1493 
1494 	rcu_read_lock();
1495 	prog = rcu_dereference(f->bpf_prog);
1496 	if (prog)
1497 		ret = bpf_prog_run_clear_cb(prog, skb) % num;
1498 	rcu_read_unlock();
1499 
1500 	return ret;
1501 }
1502 
fanout_has_flag(struct packet_fanout * f,u16 flag)1503 static bool fanout_has_flag(struct packet_fanout *f, u16 flag)
1504 {
1505 	return f->flags & (flag >> 8);
1506 }
1507 
packet_rcv_fanout(struct sk_buff * skb,struct net_device * dev,struct packet_type * pt,struct net_device * orig_dev)1508 static int packet_rcv_fanout(struct sk_buff *skb, struct net_device *dev,
1509 			     struct packet_type *pt, struct net_device *orig_dev)
1510 {
1511 	struct packet_fanout *f = pt->af_packet_priv;
1512 	unsigned int num = READ_ONCE(f->num_members);
1513 	struct net *net = read_pnet(&f->net);
1514 	struct packet_sock *po;
1515 	unsigned int idx;
1516 
1517 	if (!net_eq(dev_net(dev), net) || !num) {
1518 		kfree_skb(skb);
1519 		return 0;
1520 	}
1521 
1522 	if (fanout_has_flag(f, PACKET_FANOUT_FLAG_DEFRAG)) {
1523 		skb = ip_check_defrag(net, skb, IP_DEFRAG_AF_PACKET);
1524 		if (!skb)
1525 			return 0;
1526 	}
1527 	switch (f->type) {
1528 	case PACKET_FANOUT_HASH:
1529 	default:
1530 		idx = fanout_demux_hash(f, skb, num);
1531 		break;
1532 	case PACKET_FANOUT_LB:
1533 		idx = fanout_demux_lb(f, skb, num);
1534 		break;
1535 	case PACKET_FANOUT_CPU:
1536 		idx = fanout_demux_cpu(f, skb, num);
1537 		break;
1538 	case PACKET_FANOUT_RND:
1539 		idx = fanout_demux_rnd(f, skb, num);
1540 		break;
1541 	case PACKET_FANOUT_QM:
1542 		idx = fanout_demux_qm(f, skb, num);
1543 		break;
1544 	case PACKET_FANOUT_ROLLOVER:
1545 		idx = fanout_demux_rollover(f, skb, 0, false, num);
1546 		break;
1547 	case PACKET_FANOUT_CBPF:
1548 	case PACKET_FANOUT_EBPF:
1549 		idx = fanout_demux_bpf(f, skb, num);
1550 		break;
1551 	}
1552 
1553 	if (fanout_has_flag(f, PACKET_FANOUT_FLAG_ROLLOVER))
1554 		idx = fanout_demux_rollover(f, skb, idx, true, num);
1555 
1556 	po = pkt_sk(rcu_dereference(f->arr[idx]));
1557 	return po->prot_hook.func(skb, dev, &po->prot_hook, orig_dev);
1558 }
1559 
1560 DEFINE_MUTEX(fanout_mutex);
1561 EXPORT_SYMBOL_GPL(fanout_mutex);
1562 static LIST_HEAD(fanout_list);
1563 static u16 fanout_next_id;
1564 
__fanout_link(struct sock * sk,struct packet_sock * po)1565 static void __fanout_link(struct sock *sk, struct packet_sock *po)
1566 {
1567 	struct packet_fanout *f = po->fanout;
1568 
1569 	spin_lock(&f->lock);
1570 	rcu_assign_pointer(f->arr[f->num_members], sk);
1571 	smp_wmb();
1572 	f->num_members++;
1573 	if (f->num_members == 1)
1574 		dev_add_pack(&f->prot_hook);
1575 	spin_unlock(&f->lock);
1576 }
1577 
__fanout_unlink(struct sock * sk,struct packet_sock * po)1578 static void __fanout_unlink(struct sock *sk, struct packet_sock *po)
1579 {
1580 	struct packet_fanout *f = po->fanout;
1581 	int i;
1582 
1583 	spin_lock(&f->lock);
1584 	for (i = 0; i < f->num_members; i++) {
1585 		if (rcu_dereference_protected(f->arr[i],
1586 					      lockdep_is_held(&f->lock)) == sk)
1587 			break;
1588 	}
1589 	BUG_ON(i >= f->num_members);
1590 	rcu_assign_pointer(f->arr[i],
1591 			   rcu_dereference_protected(f->arr[f->num_members - 1],
1592 						     lockdep_is_held(&f->lock)));
1593 	f->num_members--;
1594 	if (f->num_members == 0)
1595 		__dev_remove_pack(&f->prot_hook);
1596 	spin_unlock(&f->lock);
1597 }
1598 
match_fanout_group(struct packet_type * ptype,struct sock * sk)1599 static bool match_fanout_group(struct packet_type *ptype, struct sock *sk)
1600 {
1601 	if (sk->sk_family != PF_PACKET)
1602 		return false;
1603 
1604 	return ptype->af_packet_priv == pkt_sk(sk)->fanout;
1605 }
1606 
fanout_init_data(struct packet_fanout * f)1607 static void fanout_init_data(struct packet_fanout *f)
1608 {
1609 	switch (f->type) {
1610 	case PACKET_FANOUT_LB:
1611 		atomic_set(&f->rr_cur, 0);
1612 		break;
1613 	case PACKET_FANOUT_CBPF:
1614 	case PACKET_FANOUT_EBPF:
1615 		RCU_INIT_POINTER(f->bpf_prog, NULL);
1616 		break;
1617 	}
1618 }
1619 
__fanout_set_data_bpf(struct packet_fanout * f,struct bpf_prog * new)1620 static void __fanout_set_data_bpf(struct packet_fanout *f, struct bpf_prog *new)
1621 {
1622 	struct bpf_prog *old;
1623 
1624 	spin_lock(&f->lock);
1625 	old = rcu_dereference_protected(f->bpf_prog, lockdep_is_held(&f->lock));
1626 	rcu_assign_pointer(f->bpf_prog, new);
1627 	spin_unlock(&f->lock);
1628 
1629 	if (old) {
1630 		synchronize_net();
1631 		bpf_prog_destroy(old);
1632 	}
1633 }
1634 
fanout_set_data_cbpf(struct packet_sock * po,sockptr_t data,unsigned int len)1635 static int fanout_set_data_cbpf(struct packet_sock *po, sockptr_t data,
1636 				unsigned int len)
1637 {
1638 	struct bpf_prog *new;
1639 	struct sock_fprog fprog;
1640 	int ret;
1641 
1642 	if (sock_flag(&po->sk, SOCK_FILTER_LOCKED))
1643 		return -EPERM;
1644 
1645 	ret = copy_bpf_fprog_from_user(&fprog, data, len);
1646 	if (ret)
1647 		return ret;
1648 
1649 	ret = bpf_prog_create_from_user(&new, &fprog, NULL, false);
1650 	if (ret)
1651 		return ret;
1652 
1653 	__fanout_set_data_bpf(po->fanout, new);
1654 	return 0;
1655 }
1656 
fanout_set_data_ebpf(struct packet_sock * po,sockptr_t data,unsigned int len)1657 static int fanout_set_data_ebpf(struct packet_sock *po, sockptr_t data,
1658 				unsigned int len)
1659 {
1660 	struct bpf_prog *new;
1661 	u32 fd;
1662 
1663 	if (sock_flag(&po->sk, SOCK_FILTER_LOCKED))
1664 		return -EPERM;
1665 	if (len != sizeof(fd))
1666 		return -EINVAL;
1667 	if (copy_from_sockptr(&fd, data, len))
1668 		return -EFAULT;
1669 
1670 	new = bpf_prog_get_type(fd, BPF_PROG_TYPE_SOCKET_FILTER);
1671 	if (IS_ERR(new))
1672 		return PTR_ERR(new);
1673 
1674 	__fanout_set_data_bpf(po->fanout, new);
1675 	return 0;
1676 }
1677 
fanout_set_data(struct packet_sock * po,sockptr_t data,unsigned int len)1678 static int fanout_set_data(struct packet_sock *po, sockptr_t data,
1679 			   unsigned int len)
1680 {
1681 	switch (po->fanout->type) {
1682 	case PACKET_FANOUT_CBPF:
1683 		return fanout_set_data_cbpf(po, data, len);
1684 	case PACKET_FANOUT_EBPF:
1685 		return fanout_set_data_ebpf(po, data, len);
1686 	default:
1687 		return -EINVAL;
1688 	}
1689 }
1690 
fanout_release_data(struct packet_fanout * f)1691 static void fanout_release_data(struct packet_fanout *f)
1692 {
1693 	switch (f->type) {
1694 	case PACKET_FANOUT_CBPF:
1695 	case PACKET_FANOUT_EBPF:
1696 		__fanout_set_data_bpf(f, NULL);
1697 	}
1698 }
1699 
__fanout_id_is_free(struct sock * sk,u16 candidate_id)1700 static bool __fanout_id_is_free(struct sock *sk, u16 candidate_id)
1701 {
1702 	struct packet_fanout *f;
1703 
1704 	list_for_each_entry(f, &fanout_list, list) {
1705 		if (f->id == candidate_id &&
1706 		    read_pnet(&f->net) == sock_net(sk)) {
1707 			return false;
1708 		}
1709 	}
1710 	return true;
1711 }
1712 
fanout_find_new_id(struct sock * sk,u16 * new_id)1713 static bool fanout_find_new_id(struct sock *sk, u16 *new_id)
1714 {
1715 	u16 id = fanout_next_id;
1716 
1717 	do {
1718 		if (__fanout_id_is_free(sk, id)) {
1719 			*new_id = id;
1720 			fanout_next_id = id + 1;
1721 			return true;
1722 		}
1723 
1724 		id++;
1725 	} while (id != fanout_next_id);
1726 
1727 	return false;
1728 }
1729 
fanout_add(struct sock * sk,struct fanout_args * args)1730 static int fanout_add(struct sock *sk, struct fanout_args *args)
1731 {
1732 	struct packet_rollover *rollover = NULL;
1733 	struct packet_sock *po = pkt_sk(sk);
1734 	u16 type_flags = args->type_flags;
1735 	struct packet_fanout *f, *match;
1736 	u8 type = type_flags & 0xff;
1737 	u8 flags = type_flags >> 8;
1738 	u16 id = args->id;
1739 	int err;
1740 
1741 	switch (type) {
1742 	case PACKET_FANOUT_ROLLOVER:
1743 		if (type_flags & PACKET_FANOUT_FLAG_ROLLOVER)
1744 			return -EINVAL;
1745 		break;
1746 	case PACKET_FANOUT_HASH:
1747 	case PACKET_FANOUT_LB:
1748 	case PACKET_FANOUT_CPU:
1749 	case PACKET_FANOUT_RND:
1750 	case PACKET_FANOUT_QM:
1751 	case PACKET_FANOUT_CBPF:
1752 	case PACKET_FANOUT_EBPF:
1753 		break;
1754 	default:
1755 		return -EINVAL;
1756 	}
1757 
1758 	mutex_lock(&fanout_mutex);
1759 
1760 	err = -EALREADY;
1761 	if (po->fanout)
1762 		goto out;
1763 
1764 	if (type == PACKET_FANOUT_ROLLOVER ||
1765 	    (type_flags & PACKET_FANOUT_FLAG_ROLLOVER)) {
1766 		err = -ENOMEM;
1767 		rollover = kzalloc(sizeof(*rollover), GFP_KERNEL);
1768 		if (!rollover)
1769 			goto out;
1770 		atomic_long_set(&rollover->num, 0);
1771 		atomic_long_set(&rollover->num_huge, 0);
1772 		atomic_long_set(&rollover->num_failed, 0);
1773 	}
1774 
1775 	if (type_flags & PACKET_FANOUT_FLAG_UNIQUEID) {
1776 		if (id != 0) {
1777 			err = -EINVAL;
1778 			goto out;
1779 		}
1780 		if (!fanout_find_new_id(sk, &id)) {
1781 			err = -ENOMEM;
1782 			goto out;
1783 		}
1784 		/* ephemeral flag for the first socket in the group: drop it */
1785 		flags &= ~(PACKET_FANOUT_FLAG_UNIQUEID >> 8);
1786 	}
1787 
1788 	match = NULL;
1789 	list_for_each_entry(f, &fanout_list, list) {
1790 		if (f->id == id &&
1791 		    read_pnet(&f->net) == sock_net(sk)) {
1792 			match = f;
1793 			break;
1794 		}
1795 	}
1796 	err = -EINVAL;
1797 	if (match) {
1798 		if (match->flags != flags)
1799 			goto out;
1800 		if (args->max_num_members &&
1801 		    args->max_num_members != match->max_num_members)
1802 			goto out;
1803 	} else {
1804 		if (args->max_num_members > PACKET_FANOUT_MAX)
1805 			goto out;
1806 		if (!args->max_num_members)
1807 			/* legacy PACKET_FANOUT_MAX */
1808 			args->max_num_members = 256;
1809 		err = -ENOMEM;
1810 		match = kvzalloc(struct_size(match, arr, args->max_num_members),
1811 				 GFP_KERNEL);
1812 		if (!match)
1813 			goto out;
1814 		write_pnet(&match->net, sock_net(sk));
1815 		match->id = id;
1816 		match->type = type;
1817 		match->flags = flags;
1818 		INIT_LIST_HEAD(&match->list);
1819 		spin_lock_init(&match->lock);
1820 		refcount_set(&match->sk_ref, 0);
1821 		fanout_init_data(match);
1822 		match->prot_hook.type = po->prot_hook.type;
1823 		match->prot_hook.dev = po->prot_hook.dev;
1824 		match->prot_hook.func = packet_rcv_fanout;
1825 		match->prot_hook.af_packet_priv = match;
1826 		match->prot_hook.af_packet_net = read_pnet(&match->net);
1827 		match->prot_hook.id_match = match_fanout_group;
1828 		match->max_num_members = args->max_num_members;
1829 		match->prot_hook.ignore_outgoing = type_flags & PACKET_FANOUT_FLAG_IGNORE_OUTGOING;
1830 		list_add(&match->list, &fanout_list);
1831 	}
1832 	err = -EINVAL;
1833 
1834 	spin_lock(&po->bind_lock);
1835 	if (packet_sock_flag(po, PACKET_SOCK_RUNNING) &&
1836 	    match->type == type &&
1837 	    match->prot_hook.type == po->prot_hook.type &&
1838 	    match->prot_hook.dev == po->prot_hook.dev) {
1839 		err = -ENOSPC;
1840 		if (refcount_read(&match->sk_ref) < match->max_num_members) {
1841 			__dev_remove_pack(&po->prot_hook);
1842 
1843 			/* Paired with packet_setsockopt(PACKET_FANOUT_DATA) */
1844 			WRITE_ONCE(po->fanout, match);
1845 
1846 			po->rollover = rollover;
1847 			rollover = NULL;
1848 			refcount_set(&match->sk_ref, refcount_read(&match->sk_ref) + 1);
1849 			__fanout_link(sk, po);
1850 			err = 0;
1851 		}
1852 	}
1853 	spin_unlock(&po->bind_lock);
1854 
1855 	if (err && !refcount_read(&match->sk_ref)) {
1856 		list_del(&match->list);
1857 		kvfree(match);
1858 	}
1859 
1860 out:
1861 	kfree(rollover);
1862 	mutex_unlock(&fanout_mutex);
1863 	return err;
1864 }
1865 
1866 /* If pkt_sk(sk)->fanout->sk_ref is zero, this function removes
1867  * pkt_sk(sk)->fanout from fanout_list and returns pkt_sk(sk)->fanout.
1868  * It is the responsibility of the caller to call fanout_release_data() and
1869  * free the returned packet_fanout (after synchronize_net())
1870  */
fanout_release(struct sock * sk)1871 static struct packet_fanout *fanout_release(struct sock *sk)
1872 {
1873 	struct packet_sock *po = pkt_sk(sk);
1874 	struct packet_fanout *f;
1875 
1876 	mutex_lock(&fanout_mutex);
1877 	f = po->fanout;
1878 	if (f) {
1879 		po->fanout = NULL;
1880 
1881 		if (refcount_dec_and_test(&f->sk_ref))
1882 			list_del(&f->list);
1883 		else
1884 			f = NULL;
1885 	}
1886 	mutex_unlock(&fanout_mutex);
1887 
1888 	return f;
1889 }
1890 
packet_extra_vlan_len_allowed(const struct net_device * dev,struct sk_buff * skb)1891 static bool packet_extra_vlan_len_allowed(const struct net_device *dev,
1892 					  struct sk_buff *skb)
1893 {
1894 	/* Earlier code assumed this would be a VLAN pkt, double-check
1895 	 * this now that we have the actual packet in hand. We can only
1896 	 * do this check on Ethernet devices.
1897 	 */
1898 	if (unlikely(dev->type != ARPHRD_ETHER))
1899 		return false;
1900 
1901 	skb_reset_mac_header(skb);
1902 	return likely(eth_hdr(skb)->h_proto == htons(ETH_P_8021Q));
1903 }
1904 
1905 static const struct proto_ops packet_ops;
1906 
1907 static const struct proto_ops packet_ops_spkt;
1908 
packet_rcv_spkt(struct sk_buff * skb,struct net_device * dev,struct packet_type * pt,struct net_device * orig_dev)1909 static int packet_rcv_spkt(struct sk_buff *skb, struct net_device *dev,
1910 			   struct packet_type *pt, struct net_device *orig_dev)
1911 {
1912 	struct sock *sk;
1913 	struct sockaddr_pkt *spkt;
1914 
1915 	/*
1916 	 *	When we registered the protocol we saved the socket in the data
1917 	 *	field for just this event.
1918 	 */
1919 
1920 	sk = pt->af_packet_priv;
1921 
1922 	/*
1923 	 *	Yank back the headers [hope the device set this
1924 	 *	right or kerboom...]
1925 	 *
1926 	 *	Incoming packets have ll header pulled,
1927 	 *	push it back.
1928 	 *
1929 	 *	For outgoing ones skb->data == skb_mac_header(skb)
1930 	 *	so that this procedure is noop.
1931 	 */
1932 
1933 	if (skb->pkt_type == PACKET_LOOPBACK)
1934 		goto out;
1935 
1936 	if (!net_eq(dev_net(dev), sock_net(sk)))
1937 		goto out;
1938 
1939 	skb = skb_share_check(skb, GFP_ATOMIC);
1940 	if (skb == NULL)
1941 		goto oom;
1942 
1943 	/* drop any routing info */
1944 	skb_dst_drop(skb);
1945 
1946 	/* drop conntrack reference */
1947 	nf_reset_ct(skb);
1948 
1949 	spkt = &PACKET_SKB_CB(skb)->sa.pkt;
1950 
1951 	skb_push(skb, skb->data - skb_mac_header(skb));
1952 
1953 	/*
1954 	 *	The SOCK_PACKET socket receives _all_ frames.
1955 	 */
1956 
1957 	spkt->spkt_family = dev->type;
1958 	strscpy(spkt->spkt_device, dev->name, sizeof(spkt->spkt_device));
1959 	spkt->spkt_protocol = skb->protocol;
1960 
1961 	/*
1962 	 *	Charge the memory to the socket. This is done specifically
1963 	 *	to prevent sockets using all the memory up.
1964 	 */
1965 
1966 	if (sock_queue_rcv_skb(sk, skb) == 0)
1967 		return 0;
1968 
1969 out:
1970 	kfree_skb(skb);
1971 oom:
1972 	return 0;
1973 }
1974 
packet_parse_headers(struct sk_buff * skb,struct socket * sock)1975 static void packet_parse_headers(struct sk_buff *skb, struct socket *sock)
1976 {
1977 	int depth;
1978 
1979 	if ((!skb->protocol || skb->protocol == htons(ETH_P_ALL)) &&
1980 	    sock->type == SOCK_RAW) {
1981 		skb_reset_mac_header(skb);
1982 		skb->protocol = dev_parse_header_protocol(skb);
1983 	}
1984 
1985 	/* Move network header to the right position for VLAN tagged packets */
1986 	if (likely(skb->dev->type == ARPHRD_ETHER) &&
1987 	    eth_type_vlan(skb->protocol) &&
1988 	    vlan_get_protocol_and_depth(skb, skb->protocol, &depth) != 0)
1989 		skb_set_network_header(skb, depth);
1990 
1991 	skb_probe_transport_header(skb);
1992 }
1993 
1994 /*
1995  *	Output a raw packet to a device layer. This bypasses all the other
1996  *	protocol layers and you must therefore supply it with a complete frame
1997  */
1998 
packet_sendmsg_spkt(struct socket * sock,struct msghdr * msg,size_t len)1999 static int packet_sendmsg_spkt(struct socket *sock, struct msghdr *msg,
2000 			       size_t len)
2001 {
2002 	struct sock *sk = sock->sk;
2003 	DECLARE_SOCKADDR(struct sockaddr_pkt *, saddr, msg->msg_name);
2004 	struct sk_buff *skb = NULL;
2005 	struct net_device *dev;
2006 	struct sockcm_cookie sockc;
2007 	__be16 proto = 0;
2008 	int err;
2009 	int extra_len = 0;
2010 
2011 	/*
2012 	 *	Get and verify the address.
2013 	 */
2014 
2015 	if (saddr) {
2016 		if (msg->msg_namelen < sizeof(struct sockaddr))
2017 			return -EINVAL;
2018 		if (msg->msg_namelen == sizeof(struct sockaddr_pkt))
2019 			proto = saddr->spkt_protocol;
2020 	} else
2021 		return -ENOTCONN;	/* SOCK_PACKET must be sent giving an address */
2022 
2023 	/*
2024 	 *	Find the device first to size check it
2025 	 */
2026 
2027 	saddr->spkt_device[sizeof(saddr->spkt_device) - 1] = 0;
2028 retry:
2029 	rcu_read_lock();
2030 	dev = dev_get_by_name_rcu(sock_net(sk), saddr->spkt_device);
2031 	err = -ENODEV;
2032 	if (dev == NULL)
2033 		goto out_unlock;
2034 
2035 	err = -ENETDOWN;
2036 	if (!(dev->flags & IFF_UP))
2037 		goto out_unlock;
2038 
2039 	/*
2040 	 * You may not queue a frame bigger than the mtu. This is the lowest level
2041 	 * raw protocol and you must do your own fragmentation at this level.
2042 	 */
2043 
2044 	if (unlikely(sock_flag(sk, SOCK_NOFCS))) {
2045 		if (!netif_supports_nofcs(dev)) {
2046 			err = -EPROTONOSUPPORT;
2047 			goto out_unlock;
2048 		}
2049 		extra_len = 4; /* We're doing our own CRC */
2050 	}
2051 
2052 	err = -EMSGSIZE;
2053 	if (len > dev->mtu + dev->hard_header_len + VLAN_HLEN + extra_len)
2054 		goto out_unlock;
2055 
2056 	if (!skb) {
2057 		size_t reserved = LL_RESERVED_SPACE(dev);
2058 		int tlen = dev->needed_tailroom;
2059 		unsigned int hhlen = dev->header_ops ? dev->hard_header_len : 0;
2060 
2061 		rcu_read_unlock();
2062 		skb = sock_wmalloc(sk, len + reserved + tlen, 0, GFP_KERNEL);
2063 		if (skb == NULL)
2064 			return -ENOBUFS;
2065 		/* FIXME: Save some space for broken drivers that write a hard
2066 		 * header at transmission time by themselves. PPP is the notable
2067 		 * one here. This should really be fixed at the driver level.
2068 		 */
2069 		skb_reserve(skb, reserved);
2070 		skb_reset_network_header(skb);
2071 
2072 		/* Try to align data part correctly */
2073 		if (hhlen) {
2074 			skb->data -= hhlen;
2075 			skb->tail -= hhlen;
2076 			if (len < hhlen)
2077 				skb_reset_network_header(skb);
2078 		}
2079 		err = memcpy_from_msg(skb_put(skb, len), msg, len);
2080 		if (err)
2081 			goto out_free;
2082 		goto retry;
2083 	}
2084 
2085 	if (!dev_validate_header(dev, skb->data, len) || !skb->len) {
2086 		err = -EINVAL;
2087 		goto out_unlock;
2088 	}
2089 	if (len > (dev->mtu + dev->hard_header_len + extra_len) &&
2090 	    !packet_extra_vlan_len_allowed(dev, skb)) {
2091 		err = -EMSGSIZE;
2092 		goto out_unlock;
2093 	}
2094 
2095 	sockcm_init(&sockc, sk);
2096 	if (msg->msg_controllen) {
2097 		err = sock_cmsg_send(sk, msg, &sockc);
2098 		if (unlikely(err))
2099 			goto out_unlock;
2100 	}
2101 
2102 	skb->protocol = proto;
2103 	skb->dev = dev;
2104 	skb->priority = READ_ONCE(sk->sk_priority);
2105 	skb->mark = READ_ONCE(sk->sk_mark);
2106 	skb->tstamp = sockc.transmit_time;
2107 
2108 	skb_setup_tx_timestamp(skb, sockc.tsflags);
2109 
2110 	if (unlikely(extra_len == 4))
2111 		skb->no_fcs = 1;
2112 
2113 	packet_parse_headers(skb, sock);
2114 
2115 	dev_queue_xmit(skb);
2116 	rcu_read_unlock();
2117 	return len;
2118 
2119 out_unlock:
2120 	rcu_read_unlock();
2121 out_free:
2122 	kfree_skb(skb);
2123 	return err;
2124 }
2125 
run_filter(struct sk_buff * skb,const struct sock * sk,unsigned int res)2126 static unsigned int run_filter(struct sk_buff *skb,
2127 			       const struct sock *sk,
2128 			       unsigned int res)
2129 {
2130 	struct sk_filter *filter;
2131 
2132 	rcu_read_lock();
2133 	filter = rcu_dereference(sk->sk_filter);
2134 	if (filter != NULL)
2135 		res = bpf_prog_run_clear_cb(filter->prog, skb);
2136 	rcu_read_unlock();
2137 
2138 	return res;
2139 }
2140 
packet_rcv_vnet(struct msghdr * msg,const struct sk_buff * skb,size_t * len,int vnet_hdr_sz)2141 static int packet_rcv_vnet(struct msghdr *msg, const struct sk_buff *skb,
2142 			   size_t *len, int vnet_hdr_sz)
2143 {
2144 	struct virtio_net_hdr_mrg_rxbuf vnet_hdr = { .num_buffers = 0 };
2145 
2146 	if (*len < vnet_hdr_sz)
2147 		return -EINVAL;
2148 	*len -= vnet_hdr_sz;
2149 
2150 	if (virtio_net_hdr_from_skb(skb, (struct virtio_net_hdr *)&vnet_hdr, vio_le(), true, 0))
2151 		return -EINVAL;
2152 
2153 	return memcpy_to_msg(msg, (void *)&vnet_hdr, vnet_hdr_sz);
2154 }
2155 
2156 /*
2157  * This function makes lazy skb cloning in hope that most of packets
2158  * are discarded by BPF.
2159  *
2160  * Note tricky part: we DO mangle shared skb! skb->data, skb->len
2161  * and skb->cb are mangled. It works because (and until) packets
2162  * falling here are owned by current CPU. Output packets are cloned
2163  * by dev_queue_xmit_nit(), input packets are processed by net_bh
2164  * sequentially, so that if we return skb to original state on exit,
2165  * we will not harm anyone.
2166  */
2167 
packet_rcv(struct sk_buff * skb,struct net_device * dev,struct packet_type * pt,struct net_device * orig_dev)2168 static int packet_rcv(struct sk_buff *skb, struct net_device *dev,
2169 		      struct packet_type *pt, struct net_device *orig_dev)
2170 {
2171 	struct sock *sk;
2172 	struct sockaddr_ll *sll;
2173 	struct packet_sock *po;
2174 	u8 *skb_head = skb->data;
2175 	int skb_len = skb->len;
2176 	unsigned int snaplen, res;
2177 	bool is_drop_n_account = false;
2178 
2179 	if (skb->pkt_type == PACKET_LOOPBACK)
2180 		goto drop;
2181 
2182 	sk = pt->af_packet_priv;
2183 	po = pkt_sk(sk);
2184 
2185 	if (!net_eq(dev_net(dev), sock_net(sk)))
2186 		goto drop;
2187 
2188 	skb->dev = dev;
2189 
2190 	if (dev_has_header(dev)) {
2191 		/* The device has an explicit notion of ll header,
2192 		 * exported to higher levels.
2193 		 *
2194 		 * Otherwise, the device hides details of its frame
2195 		 * structure, so that corresponding packet head is
2196 		 * never delivered to user.
2197 		 */
2198 		if (sk->sk_type != SOCK_DGRAM)
2199 			skb_push(skb, skb->data - skb_mac_header(skb));
2200 		else if (skb->pkt_type == PACKET_OUTGOING) {
2201 			/* Special case: outgoing packets have ll header at head */
2202 			skb_pull(skb, skb_network_offset(skb));
2203 		}
2204 	}
2205 
2206 	snaplen = skb->len;
2207 
2208 	res = run_filter(skb, sk, snaplen);
2209 	if (!res)
2210 		goto drop_n_restore;
2211 	if (snaplen > res)
2212 		snaplen = res;
2213 
2214 	if (atomic_read(&sk->sk_rmem_alloc) >= sk->sk_rcvbuf)
2215 		goto drop_n_acct;
2216 
2217 	if (skb_shared(skb)) {
2218 		struct sk_buff *nskb = skb_clone(skb, GFP_ATOMIC);
2219 		if (nskb == NULL)
2220 			goto drop_n_acct;
2221 
2222 		if (skb_head != skb->data) {
2223 			skb->data = skb_head;
2224 			skb->len = skb_len;
2225 		}
2226 		consume_skb(skb);
2227 		skb = nskb;
2228 	}
2229 
2230 	sock_skb_cb_check_size(sizeof(*PACKET_SKB_CB(skb)) + MAX_ADDR_LEN - 8);
2231 
2232 	sll = &PACKET_SKB_CB(skb)->sa.ll;
2233 	sll->sll_hatype = dev->type;
2234 	sll->sll_pkttype = skb->pkt_type;
2235 	if (unlikely(packet_sock_flag(po, PACKET_SOCK_ORIGDEV)))
2236 		sll->sll_ifindex = orig_dev->ifindex;
2237 	else
2238 		sll->sll_ifindex = dev->ifindex;
2239 
2240 	sll->sll_halen = dev_parse_header(skb, sll->sll_addr);
2241 
2242 	/* sll->sll_family and sll->sll_protocol are set in packet_recvmsg().
2243 	 * Use their space for storing the original skb length.
2244 	 */
2245 	PACKET_SKB_CB(skb)->sa.origlen = skb->len;
2246 
2247 	if (pskb_trim(skb, snaplen))
2248 		goto drop_n_acct;
2249 
2250 	skb_set_owner_r(skb, sk);
2251 	skb->dev = NULL;
2252 	skb_dst_drop(skb);
2253 
2254 	/* drop conntrack reference */
2255 	nf_reset_ct(skb);
2256 
2257 	spin_lock(&sk->sk_receive_queue.lock);
2258 	po->stats.stats1.tp_packets++;
2259 	sock_skb_set_dropcount(sk, skb);
2260 	skb_clear_delivery_time(skb);
2261 	__skb_queue_tail(&sk->sk_receive_queue, skb);
2262 	spin_unlock(&sk->sk_receive_queue.lock);
2263 	sk->sk_data_ready(sk);
2264 	return 0;
2265 
2266 drop_n_acct:
2267 	is_drop_n_account = true;
2268 	atomic_inc(&po->tp_drops);
2269 	atomic_inc(&sk->sk_drops);
2270 
2271 drop_n_restore:
2272 	if (skb_head != skb->data && skb_shared(skb)) {
2273 		skb->data = skb_head;
2274 		skb->len = skb_len;
2275 	}
2276 drop:
2277 	if (!is_drop_n_account)
2278 		consume_skb(skb);
2279 	else
2280 		kfree_skb(skb);
2281 	return 0;
2282 }
2283 
tpacket_rcv(struct sk_buff * skb,struct net_device * dev,struct packet_type * pt,struct net_device * orig_dev)2284 static int tpacket_rcv(struct sk_buff *skb, struct net_device *dev,
2285 		       struct packet_type *pt, struct net_device *orig_dev)
2286 {
2287 	struct sock *sk;
2288 	struct packet_sock *po;
2289 	struct sockaddr_ll *sll;
2290 	union tpacket_uhdr h;
2291 	u8 *skb_head = skb->data;
2292 	int skb_len = skb->len;
2293 	unsigned int snaplen, res;
2294 	unsigned long status = TP_STATUS_USER;
2295 	unsigned short macoff, hdrlen;
2296 	unsigned int netoff;
2297 	struct sk_buff *copy_skb = NULL;
2298 	struct timespec64 ts;
2299 	__u32 ts_status;
2300 	bool is_drop_n_account = false;
2301 	unsigned int slot_id = 0;
2302 	int vnet_hdr_sz = 0;
2303 
2304 	/* struct tpacket{2,3}_hdr is aligned to a multiple of TPACKET_ALIGNMENT.
2305 	 * We may add members to them until current aligned size without forcing
2306 	 * userspace to call getsockopt(..., PACKET_HDRLEN, ...).
2307 	 */
2308 	BUILD_BUG_ON(TPACKET_ALIGN(sizeof(*h.h2)) != 32);
2309 	BUILD_BUG_ON(TPACKET_ALIGN(sizeof(*h.h3)) != 48);
2310 
2311 	if (skb->pkt_type == PACKET_LOOPBACK)
2312 		goto drop;
2313 
2314 	sk = pt->af_packet_priv;
2315 	po = pkt_sk(sk);
2316 
2317 	if (!net_eq(dev_net(dev), sock_net(sk)))
2318 		goto drop;
2319 
2320 	if (dev_has_header(dev)) {
2321 		if (sk->sk_type != SOCK_DGRAM)
2322 			skb_push(skb, skb->data - skb_mac_header(skb));
2323 		else if (skb->pkt_type == PACKET_OUTGOING) {
2324 			/* Special case: outgoing packets have ll header at head */
2325 			skb_pull(skb, skb_network_offset(skb));
2326 		}
2327 	}
2328 
2329 	snaplen = skb->len;
2330 
2331 	res = run_filter(skb, sk, snaplen);
2332 	if (!res)
2333 		goto drop_n_restore;
2334 
2335 	/* If we are flooded, just give up */
2336 	if (__packet_rcv_has_room(po, skb) == ROOM_NONE) {
2337 		atomic_inc(&po->tp_drops);
2338 		goto drop_n_restore;
2339 	}
2340 
2341 	if (skb->ip_summed == CHECKSUM_PARTIAL)
2342 		status |= TP_STATUS_CSUMNOTREADY;
2343 	else if (skb->pkt_type != PACKET_OUTGOING &&
2344 		 skb_csum_unnecessary(skb))
2345 		status |= TP_STATUS_CSUM_VALID;
2346 	if (skb_is_gso(skb) && skb_is_gso_tcp(skb))
2347 		status |= TP_STATUS_GSO_TCP;
2348 
2349 	if (snaplen > res)
2350 		snaplen = res;
2351 
2352 	if (sk->sk_type == SOCK_DGRAM) {
2353 		macoff = netoff = TPACKET_ALIGN(po->tp_hdrlen) + 16 +
2354 				  po->tp_reserve;
2355 	} else {
2356 		unsigned int maclen = skb_network_offset(skb);
2357 		netoff = TPACKET_ALIGN(po->tp_hdrlen +
2358 				       (maclen < 16 ? 16 : maclen)) +
2359 				       po->tp_reserve;
2360 		vnet_hdr_sz = READ_ONCE(po->vnet_hdr_sz);
2361 		if (vnet_hdr_sz)
2362 			netoff += vnet_hdr_sz;
2363 		macoff = netoff - maclen;
2364 	}
2365 	if (netoff > USHRT_MAX) {
2366 		atomic_inc(&po->tp_drops);
2367 		goto drop_n_restore;
2368 	}
2369 	if (po->tp_version <= TPACKET_V2) {
2370 		if (macoff + snaplen > po->rx_ring.frame_size) {
2371 			if (po->copy_thresh &&
2372 			    atomic_read(&sk->sk_rmem_alloc) < sk->sk_rcvbuf) {
2373 				if (skb_shared(skb)) {
2374 					copy_skb = skb_clone(skb, GFP_ATOMIC);
2375 				} else {
2376 					copy_skb = skb_get(skb);
2377 					skb_head = skb->data;
2378 				}
2379 				if (copy_skb) {
2380 					memset(&PACKET_SKB_CB(copy_skb)->sa.ll, 0,
2381 					       sizeof(PACKET_SKB_CB(copy_skb)->sa.ll));
2382 					skb_set_owner_r(copy_skb, sk);
2383 				}
2384 			}
2385 			snaplen = po->rx_ring.frame_size - macoff;
2386 			if ((int)snaplen < 0) {
2387 				snaplen = 0;
2388 				vnet_hdr_sz = 0;
2389 			}
2390 		}
2391 	} else if (unlikely(macoff + snaplen >
2392 			    GET_PBDQC_FROM_RB(&po->rx_ring)->max_frame_len)) {
2393 		u32 nval;
2394 
2395 		nval = GET_PBDQC_FROM_RB(&po->rx_ring)->max_frame_len - macoff;
2396 		pr_err_once("tpacket_rcv: packet too big, clamped from %u to %u. macoff=%u\n",
2397 			    snaplen, nval, macoff);
2398 		snaplen = nval;
2399 		if (unlikely((int)snaplen < 0)) {
2400 			snaplen = 0;
2401 			macoff = GET_PBDQC_FROM_RB(&po->rx_ring)->max_frame_len;
2402 			vnet_hdr_sz = 0;
2403 		}
2404 	}
2405 	spin_lock(&sk->sk_receive_queue.lock);
2406 	h.raw = packet_current_rx_frame(po, skb,
2407 					TP_STATUS_KERNEL, (macoff+snaplen));
2408 	if (!h.raw)
2409 		goto drop_n_account;
2410 
2411 	if (po->tp_version <= TPACKET_V2) {
2412 		slot_id = po->rx_ring.head;
2413 		if (test_bit(slot_id, po->rx_ring.rx_owner_map))
2414 			goto drop_n_account;
2415 		__set_bit(slot_id, po->rx_ring.rx_owner_map);
2416 	}
2417 
2418 	if (vnet_hdr_sz &&
2419 	    virtio_net_hdr_from_skb(skb, h.raw + macoff -
2420 				    sizeof(struct virtio_net_hdr),
2421 				    vio_le(), true, 0)) {
2422 		if (po->tp_version == TPACKET_V3)
2423 			prb_clear_blk_fill_status(&po->rx_ring);
2424 		goto drop_n_account;
2425 	}
2426 
2427 	if (po->tp_version <= TPACKET_V2) {
2428 		packet_increment_rx_head(po, &po->rx_ring);
2429 	/*
2430 	 * LOSING will be reported till you read the stats,
2431 	 * because it's COR - Clear On Read.
2432 	 * Anyways, moving it for V1/V2 only as V3 doesn't need this
2433 	 * at packet level.
2434 	 */
2435 		if (atomic_read(&po->tp_drops))
2436 			status |= TP_STATUS_LOSING;
2437 	}
2438 
2439 	po->stats.stats1.tp_packets++;
2440 	if (copy_skb) {
2441 		status |= TP_STATUS_COPY;
2442 		skb_clear_delivery_time(copy_skb);
2443 		__skb_queue_tail(&sk->sk_receive_queue, copy_skb);
2444 	}
2445 	spin_unlock(&sk->sk_receive_queue.lock);
2446 
2447 	skb_copy_bits(skb, 0, h.raw + macoff, snaplen);
2448 
2449 	/* Always timestamp; prefer an existing software timestamp taken
2450 	 * closer to the time of capture.
2451 	 */
2452 	ts_status = tpacket_get_timestamp(skb, &ts,
2453 					  READ_ONCE(po->tp_tstamp) |
2454 					  SOF_TIMESTAMPING_SOFTWARE);
2455 	if (!ts_status)
2456 		ktime_get_real_ts64(&ts);
2457 
2458 	status |= ts_status;
2459 
2460 	switch (po->tp_version) {
2461 	case TPACKET_V1:
2462 		h.h1->tp_len = skb->len;
2463 		h.h1->tp_snaplen = snaplen;
2464 		h.h1->tp_mac = macoff;
2465 		h.h1->tp_net = netoff;
2466 		h.h1->tp_sec = ts.tv_sec;
2467 		h.h1->tp_usec = ts.tv_nsec / NSEC_PER_USEC;
2468 		hdrlen = sizeof(*h.h1);
2469 		break;
2470 	case TPACKET_V2:
2471 		h.h2->tp_len = skb->len;
2472 		h.h2->tp_snaplen = snaplen;
2473 		h.h2->tp_mac = macoff;
2474 		h.h2->tp_net = netoff;
2475 		h.h2->tp_sec = ts.tv_sec;
2476 		h.h2->tp_nsec = ts.tv_nsec;
2477 		if (skb_vlan_tag_present(skb)) {
2478 			h.h2->tp_vlan_tci = skb_vlan_tag_get(skb);
2479 			h.h2->tp_vlan_tpid = ntohs(skb->vlan_proto);
2480 			status |= TP_STATUS_VLAN_VALID | TP_STATUS_VLAN_TPID_VALID;
2481 		} else if (unlikely(sk->sk_type == SOCK_DGRAM && eth_type_vlan(skb->protocol))) {
2482 			h.h2->tp_vlan_tci = vlan_get_tci(skb, skb->dev);
2483 			h.h2->tp_vlan_tpid = ntohs(skb->protocol);
2484 			status |= TP_STATUS_VLAN_VALID | TP_STATUS_VLAN_TPID_VALID;
2485 		} else {
2486 			h.h2->tp_vlan_tci = 0;
2487 			h.h2->tp_vlan_tpid = 0;
2488 		}
2489 		memset(h.h2->tp_padding, 0, sizeof(h.h2->tp_padding));
2490 		hdrlen = sizeof(*h.h2);
2491 		break;
2492 	case TPACKET_V3:
2493 		/* tp_nxt_offset,vlan are already populated above.
2494 		 * So DONT clear those fields here
2495 		 */
2496 		h.h3->tp_status |= status;
2497 		h.h3->tp_len = skb->len;
2498 		h.h3->tp_snaplen = snaplen;
2499 		h.h3->tp_mac = macoff;
2500 		h.h3->tp_net = netoff;
2501 		h.h3->tp_sec  = ts.tv_sec;
2502 		h.h3->tp_nsec = ts.tv_nsec;
2503 		memset(h.h3->tp_padding, 0, sizeof(h.h3->tp_padding));
2504 		hdrlen = sizeof(*h.h3);
2505 		break;
2506 	default:
2507 		BUG();
2508 	}
2509 
2510 	sll = h.raw + TPACKET_ALIGN(hdrlen);
2511 	sll->sll_halen = dev_parse_header(skb, sll->sll_addr);
2512 	sll->sll_family = AF_PACKET;
2513 	sll->sll_hatype = dev->type;
2514 	sll->sll_protocol = (sk->sk_type == SOCK_DGRAM) ?
2515 		vlan_get_protocol_dgram(skb) : skb->protocol;
2516 	sll->sll_pkttype = skb->pkt_type;
2517 	if (unlikely(packet_sock_flag(po, PACKET_SOCK_ORIGDEV)))
2518 		sll->sll_ifindex = orig_dev->ifindex;
2519 	else
2520 		sll->sll_ifindex = dev->ifindex;
2521 
2522 	smp_mb();
2523 
2524 #if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE == 1
2525 	if (po->tp_version <= TPACKET_V2) {
2526 		u8 *start, *end;
2527 
2528 		end = (u8 *) PAGE_ALIGN((unsigned long) h.raw +
2529 					macoff + snaplen);
2530 
2531 		for (start = h.raw; start < end; start += PAGE_SIZE)
2532 			flush_dcache_page(pgv_to_page(start));
2533 	}
2534 	smp_wmb();
2535 #endif
2536 
2537 	if (po->tp_version <= TPACKET_V2) {
2538 		spin_lock(&sk->sk_receive_queue.lock);
2539 		__packet_set_status(po, h.raw, status);
2540 		__clear_bit(slot_id, po->rx_ring.rx_owner_map);
2541 		spin_unlock(&sk->sk_receive_queue.lock);
2542 		sk->sk_data_ready(sk);
2543 	} else if (po->tp_version == TPACKET_V3) {
2544 		prb_clear_blk_fill_status(&po->rx_ring);
2545 	}
2546 
2547 drop_n_restore:
2548 	if (skb_head != skb->data && skb_shared(skb)) {
2549 		skb->data = skb_head;
2550 		skb->len = skb_len;
2551 	}
2552 drop:
2553 	if (!is_drop_n_account)
2554 		consume_skb(skb);
2555 	else
2556 		kfree_skb(skb);
2557 	return 0;
2558 
2559 drop_n_account:
2560 	spin_unlock(&sk->sk_receive_queue.lock);
2561 	atomic_inc(&po->tp_drops);
2562 	is_drop_n_account = true;
2563 
2564 	sk->sk_data_ready(sk);
2565 	kfree_skb(copy_skb);
2566 	goto drop_n_restore;
2567 }
2568 
tpacket_destruct_skb(struct sk_buff * skb)2569 static void tpacket_destruct_skb(struct sk_buff *skb)
2570 {
2571 	struct packet_sock *po = pkt_sk(skb->sk);
2572 
2573 	if (likely(po->tx_ring.pg_vec)) {
2574 		void *ph;
2575 		__u32 ts;
2576 
2577 		ph = skb_zcopy_get_nouarg(skb);
2578 		packet_dec_pending(&po->tx_ring);
2579 
2580 		ts = __packet_set_timestamp(po, ph, skb);
2581 		__packet_set_status(po, ph, TP_STATUS_AVAILABLE | ts);
2582 
2583 		complete(&po->skb_completion);
2584 	}
2585 
2586 	sock_wfree(skb);
2587 }
2588 
__packet_snd_vnet_parse(struct virtio_net_hdr * vnet_hdr,size_t len)2589 static int __packet_snd_vnet_parse(struct virtio_net_hdr *vnet_hdr, size_t len)
2590 {
2591 	if ((vnet_hdr->flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) &&
2592 	    (__virtio16_to_cpu(vio_le(), vnet_hdr->csum_start) +
2593 	     __virtio16_to_cpu(vio_le(), vnet_hdr->csum_offset) + 2 >
2594 	      __virtio16_to_cpu(vio_le(), vnet_hdr->hdr_len)))
2595 		vnet_hdr->hdr_len = __cpu_to_virtio16(vio_le(),
2596 			 __virtio16_to_cpu(vio_le(), vnet_hdr->csum_start) +
2597 			__virtio16_to_cpu(vio_le(), vnet_hdr->csum_offset) + 2);
2598 
2599 	if (__virtio16_to_cpu(vio_le(), vnet_hdr->hdr_len) > len)
2600 		return -EINVAL;
2601 
2602 	return 0;
2603 }
2604 
packet_snd_vnet_parse(struct msghdr * msg,size_t * len,struct virtio_net_hdr * vnet_hdr,int vnet_hdr_sz)2605 static int packet_snd_vnet_parse(struct msghdr *msg, size_t *len,
2606 				 struct virtio_net_hdr *vnet_hdr, int vnet_hdr_sz)
2607 {
2608 	int ret;
2609 
2610 	if (*len < vnet_hdr_sz)
2611 		return -EINVAL;
2612 	*len -= vnet_hdr_sz;
2613 
2614 	if (!copy_from_iter_full(vnet_hdr, sizeof(*vnet_hdr), &msg->msg_iter))
2615 		return -EFAULT;
2616 
2617 	ret = __packet_snd_vnet_parse(vnet_hdr, *len);
2618 	if (ret)
2619 		return ret;
2620 
2621 	/* move iter to point to the start of mac header */
2622 	if (vnet_hdr_sz != sizeof(struct virtio_net_hdr))
2623 		iov_iter_advance(&msg->msg_iter, vnet_hdr_sz - sizeof(struct virtio_net_hdr));
2624 
2625 	return 0;
2626 }
2627 
tpacket_fill_skb(struct packet_sock * po,struct sk_buff * skb,void * frame,struct net_device * dev,void * data,int tp_len,__be16 proto,unsigned char * addr,int hlen,int copylen,const struct sockcm_cookie * sockc)2628 static int tpacket_fill_skb(struct packet_sock *po, struct sk_buff *skb,
2629 		void *frame, struct net_device *dev, void *data, int tp_len,
2630 		__be16 proto, unsigned char *addr, int hlen, int copylen,
2631 		const struct sockcm_cookie *sockc)
2632 {
2633 	union tpacket_uhdr ph;
2634 	int to_write, offset, len, nr_frags, len_max;
2635 	struct socket *sock = po->sk.sk_socket;
2636 	struct page *page;
2637 	int err;
2638 
2639 	ph.raw = frame;
2640 
2641 	skb->protocol = proto;
2642 	skb->dev = dev;
2643 	skb->priority = READ_ONCE(po->sk.sk_priority);
2644 	skb->mark = READ_ONCE(po->sk.sk_mark);
2645 	skb->tstamp = sockc->transmit_time;
2646 	skb_setup_tx_timestamp(skb, sockc->tsflags);
2647 	skb_zcopy_set_nouarg(skb, ph.raw);
2648 
2649 	skb_reserve(skb, hlen);
2650 	skb_reset_network_header(skb);
2651 
2652 	to_write = tp_len;
2653 
2654 	if (sock->type == SOCK_DGRAM) {
2655 		err = dev_hard_header(skb, dev, ntohs(proto), addr,
2656 				NULL, tp_len);
2657 		if (unlikely(err < 0))
2658 			return -EINVAL;
2659 	} else if (copylen) {
2660 		int hdrlen = min_t(int, copylen, tp_len);
2661 
2662 		skb_push(skb, dev->hard_header_len);
2663 		skb_put(skb, copylen - dev->hard_header_len);
2664 		err = skb_store_bits(skb, 0, data, hdrlen);
2665 		if (unlikely(err))
2666 			return err;
2667 		if (!dev_validate_header(dev, skb->data, hdrlen))
2668 			return -EINVAL;
2669 
2670 		data += hdrlen;
2671 		to_write -= hdrlen;
2672 	}
2673 
2674 	offset = offset_in_page(data);
2675 	len_max = PAGE_SIZE - offset;
2676 	len = ((to_write > len_max) ? len_max : to_write);
2677 
2678 	skb->data_len = to_write;
2679 	skb->len += to_write;
2680 	skb->truesize += to_write;
2681 	refcount_add(to_write, &po->sk.sk_wmem_alloc);
2682 
2683 	while (likely(to_write)) {
2684 		nr_frags = skb_shinfo(skb)->nr_frags;
2685 
2686 		if (unlikely(nr_frags >= MAX_SKB_FRAGS)) {
2687 			pr_err("Packet exceed the number of skb frags(%u)\n",
2688 			       (unsigned int)MAX_SKB_FRAGS);
2689 			return -EFAULT;
2690 		}
2691 
2692 		page = pgv_to_page(data);
2693 		data += len;
2694 		flush_dcache_page(page);
2695 		get_page(page);
2696 		skb_fill_page_desc(skb, nr_frags, page, offset, len);
2697 		to_write -= len;
2698 		offset = 0;
2699 		len_max = PAGE_SIZE;
2700 		len = ((to_write > len_max) ? len_max : to_write);
2701 	}
2702 
2703 	packet_parse_headers(skb, sock);
2704 
2705 	return tp_len;
2706 }
2707 
tpacket_parse_header(struct packet_sock * po,void * frame,int size_max,void ** data)2708 static int tpacket_parse_header(struct packet_sock *po, void *frame,
2709 				int size_max, void **data)
2710 {
2711 	union tpacket_uhdr ph;
2712 	int tp_len, off;
2713 
2714 	ph.raw = frame;
2715 
2716 	switch (po->tp_version) {
2717 	case TPACKET_V3:
2718 		if (ph.h3->tp_next_offset != 0) {
2719 			pr_warn_once("variable sized slot not supported");
2720 			return -EINVAL;
2721 		}
2722 		tp_len = ph.h3->tp_len;
2723 		break;
2724 	case TPACKET_V2:
2725 		tp_len = ph.h2->tp_len;
2726 		break;
2727 	default:
2728 		tp_len = ph.h1->tp_len;
2729 		break;
2730 	}
2731 	if (unlikely(tp_len > size_max)) {
2732 		pr_err("packet size is too long (%d > %d)\n", tp_len, size_max);
2733 		return -EMSGSIZE;
2734 	}
2735 
2736 	if (unlikely(packet_sock_flag(po, PACKET_SOCK_TX_HAS_OFF))) {
2737 		int off_min, off_max;
2738 
2739 		off_min = po->tp_hdrlen - sizeof(struct sockaddr_ll);
2740 		off_max = po->tx_ring.frame_size - tp_len;
2741 		if (po->sk.sk_type == SOCK_DGRAM) {
2742 			switch (po->tp_version) {
2743 			case TPACKET_V3:
2744 				off = ph.h3->tp_net;
2745 				break;
2746 			case TPACKET_V2:
2747 				off = ph.h2->tp_net;
2748 				break;
2749 			default:
2750 				off = ph.h1->tp_net;
2751 				break;
2752 			}
2753 		} else {
2754 			switch (po->tp_version) {
2755 			case TPACKET_V3:
2756 				off = ph.h3->tp_mac;
2757 				break;
2758 			case TPACKET_V2:
2759 				off = ph.h2->tp_mac;
2760 				break;
2761 			default:
2762 				off = ph.h1->tp_mac;
2763 				break;
2764 			}
2765 		}
2766 		if (unlikely((off < off_min) || (off_max < off)))
2767 			return -EINVAL;
2768 	} else {
2769 		off = po->tp_hdrlen - sizeof(struct sockaddr_ll);
2770 	}
2771 
2772 	*data = frame + off;
2773 	return tp_len;
2774 }
2775 
tpacket_snd(struct packet_sock * po,struct msghdr * msg)2776 static int tpacket_snd(struct packet_sock *po, struct msghdr *msg)
2777 {
2778 	struct sk_buff *skb = NULL;
2779 	struct net_device *dev;
2780 	struct virtio_net_hdr *vnet_hdr = NULL;
2781 	struct sockcm_cookie sockc;
2782 	__be16 proto;
2783 	int err, reserve = 0;
2784 	void *ph;
2785 	DECLARE_SOCKADDR(struct sockaddr_ll *, saddr, msg->msg_name);
2786 	bool need_wait = !(msg->msg_flags & MSG_DONTWAIT);
2787 	int vnet_hdr_sz = READ_ONCE(po->vnet_hdr_sz);
2788 	unsigned char *addr = NULL;
2789 	int tp_len, size_max;
2790 	void *data;
2791 	int len_sum = 0;
2792 	int status = TP_STATUS_AVAILABLE;
2793 	int hlen, tlen, copylen = 0;
2794 	long timeo;
2795 
2796 	mutex_lock(&po->pg_vec_lock);
2797 
2798 	/* packet_sendmsg() check on tx_ring.pg_vec was lockless,
2799 	 * we need to confirm it under protection of pg_vec_lock.
2800 	 */
2801 	if (unlikely(!po->tx_ring.pg_vec)) {
2802 		err = -EBUSY;
2803 		goto out;
2804 	}
2805 	if (likely(saddr == NULL)) {
2806 		dev	= packet_cached_dev_get(po);
2807 		proto	= READ_ONCE(po->num);
2808 	} else {
2809 		err = -EINVAL;
2810 		if (msg->msg_namelen < sizeof(struct sockaddr_ll))
2811 			goto out;
2812 		if (msg->msg_namelen < (saddr->sll_halen
2813 					+ offsetof(struct sockaddr_ll,
2814 						sll_addr)))
2815 			goto out;
2816 		proto	= saddr->sll_protocol;
2817 		dev = dev_get_by_index(sock_net(&po->sk), saddr->sll_ifindex);
2818 		if (po->sk.sk_socket->type == SOCK_DGRAM) {
2819 			if (dev && msg->msg_namelen < dev->addr_len +
2820 				   offsetof(struct sockaddr_ll, sll_addr))
2821 				goto out_put;
2822 			addr = saddr->sll_addr;
2823 		}
2824 	}
2825 
2826 	err = -ENXIO;
2827 	if (unlikely(dev == NULL))
2828 		goto out;
2829 	err = -ENETDOWN;
2830 	if (unlikely(!(dev->flags & IFF_UP)))
2831 		goto out_put;
2832 
2833 	sockcm_init(&sockc, &po->sk);
2834 	if (msg->msg_controllen) {
2835 		err = sock_cmsg_send(&po->sk, msg, &sockc);
2836 		if (unlikely(err))
2837 			goto out_put;
2838 	}
2839 
2840 	if (po->sk.sk_socket->type == SOCK_RAW)
2841 		reserve = dev->hard_header_len;
2842 	size_max = po->tx_ring.frame_size
2843 		- (po->tp_hdrlen - sizeof(struct sockaddr_ll));
2844 
2845 	if ((size_max > dev->mtu + reserve + VLAN_HLEN) && !vnet_hdr_sz)
2846 		size_max = dev->mtu + reserve + VLAN_HLEN;
2847 
2848 	timeo = sock_sndtimeo(&po->sk, msg->msg_flags & MSG_DONTWAIT);
2849 	reinit_completion(&po->skb_completion);
2850 
2851 	do {
2852 		ph = packet_current_frame(po, &po->tx_ring,
2853 					  TP_STATUS_SEND_REQUEST);
2854 		if (unlikely(ph == NULL)) {
2855 			/* Note: packet_read_pending() might be slow if we
2856 			 * have to call it as it's per_cpu variable, but in
2857 			 * fast-path we don't have to call it, only when ph
2858 			 * is NULL, we need to check the pending_refcnt.
2859 			 */
2860 			if (need_wait && packet_read_pending(&po->tx_ring)) {
2861 				timeo = wait_for_completion_interruptible_timeout(&po->skb_completion, timeo);
2862 				if (timeo <= 0) {
2863 					err = !timeo ? -ETIMEDOUT : -ERESTARTSYS;
2864 					goto out_put;
2865 				}
2866 				/* check for additional frames */
2867 				continue;
2868 			} else
2869 				break;
2870 		}
2871 
2872 		skb = NULL;
2873 		tp_len = tpacket_parse_header(po, ph, size_max, &data);
2874 		if (tp_len < 0)
2875 			goto tpacket_error;
2876 
2877 		status = TP_STATUS_SEND_REQUEST;
2878 		hlen = LL_RESERVED_SPACE(dev);
2879 		tlen = dev->needed_tailroom;
2880 		if (vnet_hdr_sz) {
2881 			vnet_hdr = data;
2882 			data += vnet_hdr_sz;
2883 			tp_len -= vnet_hdr_sz;
2884 			if (tp_len < 0 ||
2885 			    __packet_snd_vnet_parse(vnet_hdr, tp_len)) {
2886 				tp_len = -EINVAL;
2887 				goto tpacket_error;
2888 			}
2889 			copylen = __virtio16_to_cpu(vio_le(),
2890 						    vnet_hdr->hdr_len);
2891 		}
2892 		copylen = max_t(int, copylen, dev->hard_header_len);
2893 		skb = sock_alloc_send_skb(&po->sk,
2894 				hlen + tlen + sizeof(struct sockaddr_ll) +
2895 				(copylen - dev->hard_header_len),
2896 				!need_wait, &err);
2897 
2898 		if (unlikely(skb == NULL)) {
2899 			/* we assume the socket was initially writeable ... */
2900 			if (likely(len_sum > 0))
2901 				err = len_sum;
2902 			goto out_status;
2903 		}
2904 		tp_len = tpacket_fill_skb(po, skb, ph, dev, data, tp_len, proto,
2905 					  addr, hlen, copylen, &sockc);
2906 		if (likely(tp_len >= 0) &&
2907 		    tp_len > dev->mtu + reserve &&
2908 		    !vnet_hdr_sz &&
2909 		    !packet_extra_vlan_len_allowed(dev, skb))
2910 			tp_len = -EMSGSIZE;
2911 
2912 		if (unlikely(tp_len < 0)) {
2913 tpacket_error:
2914 			if (packet_sock_flag(po, PACKET_SOCK_TP_LOSS)) {
2915 				__packet_set_status(po, ph,
2916 						TP_STATUS_AVAILABLE);
2917 				packet_increment_head(&po->tx_ring);
2918 				kfree_skb(skb);
2919 				continue;
2920 			} else {
2921 				status = TP_STATUS_WRONG_FORMAT;
2922 				err = tp_len;
2923 				goto out_status;
2924 			}
2925 		}
2926 
2927 		if (vnet_hdr_sz) {
2928 			if (virtio_net_hdr_to_skb(skb, vnet_hdr, vio_le())) {
2929 				tp_len = -EINVAL;
2930 				goto tpacket_error;
2931 			}
2932 			virtio_net_hdr_set_proto(skb, vnet_hdr);
2933 		}
2934 
2935 		skb->destructor = tpacket_destruct_skb;
2936 		__packet_set_status(po, ph, TP_STATUS_SENDING);
2937 		packet_inc_pending(&po->tx_ring);
2938 
2939 		status = TP_STATUS_SEND_REQUEST;
2940 		err = packet_xmit(po, skb);
2941 		if (unlikely(err != 0)) {
2942 			if (err > 0)
2943 				err = net_xmit_errno(err);
2944 			if (err && __packet_get_status(po, ph) ==
2945 				   TP_STATUS_AVAILABLE) {
2946 				/* skb was destructed already */
2947 				skb = NULL;
2948 				goto out_status;
2949 			}
2950 			/*
2951 			 * skb was dropped but not destructed yet;
2952 			 * let's treat it like congestion or err < 0
2953 			 */
2954 			err = 0;
2955 		}
2956 		packet_increment_head(&po->tx_ring);
2957 		len_sum += tp_len;
2958 	} while (1);
2959 
2960 	err = len_sum;
2961 	goto out_put;
2962 
2963 out_status:
2964 	__packet_set_status(po, ph, status);
2965 	kfree_skb(skb);
2966 out_put:
2967 	dev_put(dev);
2968 out:
2969 	mutex_unlock(&po->pg_vec_lock);
2970 	return err;
2971 }
2972 
packet_alloc_skb(struct sock * sk,size_t prepad,size_t reserve,size_t len,size_t linear,int noblock,int * err)2973 static struct sk_buff *packet_alloc_skb(struct sock *sk, size_t prepad,
2974 				        size_t reserve, size_t len,
2975 				        size_t linear, int noblock,
2976 				        int *err)
2977 {
2978 	struct sk_buff *skb;
2979 
2980 	/* Under a page?  Don't bother with paged skb. */
2981 	if (prepad + len < PAGE_SIZE || !linear)
2982 		linear = len;
2983 
2984 	if (len - linear > MAX_SKB_FRAGS * (PAGE_SIZE << PAGE_ALLOC_COSTLY_ORDER))
2985 		linear = len - MAX_SKB_FRAGS * (PAGE_SIZE << PAGE_ALLOC_COSTLY_ORDER);
2986 	skb = sock_alloc_send_pskb(sk, prepad + linear, len - linear, noblock,
2987 				   err, PAGE_ALLOC_COSTLY_ORDER);
2988 	if (!skb)
2989 		return NULL;
2990 
2991 	skb_reserve(skb, reserve);
2992 	skb_put(skb, linear);
2993 	skb->data_len = len - linear;
2994 	skb->len += len - linear;
2995 
2996 	return skb;
2997 }
2998 
packet_snd(struct socket * sock,struct msghdr * msg,size_t len)2999 static int packet_snd(struct socket *sock, struct msghdr *msg, size_t len)
3000 {
3001 	struct sock *sk = sock->sk;
3002 	DECLARE_SOCKADDR(struct sockaddr_ll *, saddr, msg->msg_name);
3003 	struct sk_buff *skb;
3004 	struct net_device *dev;
3005 	__be16 proto;
3006 	unsigned char *addr = NULL;
3007 	int err, reserve = 0;
3008 	struct sockcm_cookie sockc;
3009 	struct virtio_net_hdr vnet_hdr = { 0 };
3010 	int offset = 0;
3011 	struct packet_sock *po = pkt_sk(sk);
3012 	int vnet_hdr_sz = READ_ONCE(po->vnet_hdr_sz);
3013 	int hlen, tlen, linear;
3014 	int extra_len = 0;
3015 
3016 	/*
3017 	 *	Get and verify the address.
3018 	 */
3019 
3020 	if (likely(saddr == NULL)) {
3021 		dev	= packet_cached_dev_get(po);
3022 		proto	= READ_ONCE(po->num);
3023 	} else {
3024 		err = -EINVAL;
3025 		if (msg->msg_namelen < sizeof(struct sockaddr_ll))
3026 			goto out;
3027 		if (msg->msg_namelen < (saddr->sll_halen + offsetof(struct sockaddr_ll, sll_addr)))
3028 			goto out;
3029 		proto	= saddr->sll_protocol;
3030 		dev = dev_get_by_index(sock_net(sk), saddr->sll_ifindex);
3031 		if (sock->type == SOCK_DGRAM) {
3032 			if (dev && msg->msg_namelen < dev->addr_len +
3033 				   offsetof(struct sockaddr_ll, sll_addr))
3034 				goto out_unlock;
3035 			addr = saddr->sll_addr;
3036 		}
3037 	}
3038 
3039 	err = -ENXIO;
3040 	if (unlikely(dev == NULL))
3041 		goto out_unlock;
3042 	err = -ENETDOWN;
3043 	if (unlikely(!(dev->flags & IFF_UP)))
3044 		goto out_unlock;
3045 
3046 	sockcm_init(&sockc, sk);
3047 	sockc.mark = READ_ONCE(sk->sk_mark);
3048 	if (msg->msg_controllen) {
3049 		err = sock_cmsg_send(sk, msg, &sockc);
3050 		if (unlikely(err))
3051 			goto out_unlock;
3052 	}
3053 
3054 	if (sock->type == SOCK_RAW)
3055 		reserve = dev->hard_header_len;
3056 	if (vnet_hdr_sz) {
3057 		err = packet_snd_vnet_parse(msg, &len, &vnet_hdr, vnet_hdr_sz);
3058 		if (err)
3059 			goto out_unlock;
3060 	}
3061 
3062 	if (unlikely(sock_flag(sk, SOCK_NOFCS))) {
3063 		if (!netif_supports_nofcs(dev)) {
3064 			err = -EPROTONOSUPPORT;
3065 			goto out_unlock;
3066 		}
3067 		extra_len = 4; /* We're doing our own CRC */
3068 	}
3069 
3070 	err = -EMSGSIZE;
3071 	if (!vnet_hdr.gso_type &&
3072 	    (len > dev->mtu + reserve + VLAN_HLEN + extra_len))
3073 		goto out_unlock;
3074 
3075 	err = -ENOBUFS;
3076 	hlen = LL_RESERVED_SPACE(dev);
3077 	tlen = dev->needed_tailroom;
3078 	linear = __virtio16_to_cpu(vio_le(), vnet_hdr.hdr_len);
3079 	linear = max(linear, min_t(int, len, dev->hard_header_len));
3080 	skb = packet_alloc_skb(sk, hlen + tlen, hlen, len, linear,
3081 			       msg->msg_flags & MSG_DONTWAIT, &err);
3082 	if (skb == NULL)
3083 		goto out_unlock;
3084 
3085 	skb_reset_network_header(skb);
3086 
3087 	err = -EINVAL;
3088 	if (sock->type == SOCK_DGRAM) {
3089 		offset = dev_hard_header(skb, dev, ntohs(proto), addr, NULL, len);
3090 		if (unlikely(offset < 0))
3091 			goto out_free;
3092 	} else if (reserve) {
3093 		skb_reserve(skb, -reserve);
3094 		if (len < reserve + sizeof(struct ipv6hdr) &&
3095 		    dev->min_header_len != dev->hard_header_len)
3096 			skb_reset_network_header(skb);
3097 	}
3098 
3099 	/* Returns -EFAULT on error */
3100 	err = skb_copy_datagram_from_iter(skb, offset, &msg->msg_iter, len);
3101 	if (err)
3102 		goto out_free;
3103 
3104 	if ((sock->type == SOCK_RAW &&
3105 	     !dev_validate_header(dev, skb->data, len)) || !skb->len) {
3106 		err = -EINVAL;
3107 		goto out_free;
3108 	}
3109 
3110 	skb_setup_tx_timestamp(skb, sockc.tsflags);
3111 
3112 	if (!vnet_hdr.gso_type && (len > dev->mtu + reserve + extra_len) &&
3113 	    !packet_extra_vlan_len_allowed(dev, skb)) {
3114 		err = -EMSGSIZE;
3115 		goto out_free;
3116 	}
3117 
3118 	skb->protocol = proto;
3119 	skb->dev = dev;
3120 	skb->priority = READ_ONCE(sk->sk_priority);
3121 	skb->mark = sockc.mark;
3122 	skb->tstamp = sockc.transmit_time;
3123 
3124 	if (unlikely(extra_len == 4))
3125 		skb->no_fcs = 1;
3126 
3127 	packet_parse_headers(skb, sock);
3128 
3129 	if (vnet_hdr_sz) {
3130 		err = virtio_net_hdr_to_skb(skb, &vnet_hdr, vio_le());
3131 		if (err)
3132 			goto out_free;
3133 		len += vnet_hdr_sz;
3134 		virtio_net_hdr_set_proto(skb, &vnet_hdr);
3135 	}
3136 
3137 	err = packet_xmit(po, skb);
3138 
3139 	if (unlikely(err != 0)) {
3140 		if (err > 0)
3141 			err = net_xmit_errno(err);
3142 		if (err)
3143 			goto out_unlock;
3144 	}
3145 
3146 	dev_put(dev);
3147 
3148 	return len;
3149 
3150 out_free:
3151 	kfree_skb(skb);
3152 out_unlock:
3153 	dev_put(dev);
3154 out:
3155 	return err;
3156 }
3157 
packet_sendmsg(struct socket * sock,struct msghdr * msg,size_t len)3158 static int packet_sendmsg(struct socket *sock, struct msghdr *msg, size_t len)
3159 {
3160 	struct sock *sk = sock->sk;
3161 	struct packet_sock *po = pkt_sk(sk);
3162 
3163 	/* Reading tx_ring.pg_vec without holding pg_vec_lock is racy.
3164 	 * tpacket_snd() will redo the check safely.
3165 	 */
3166 	if (data_race(po->tx_ring.pg_vec))
3167 		return tpacket_snd(po, msg);
3168 
3169 	return packet_snd(sock, msg, len);
3170 }
3171 
3172 /*
3173  *	Close a PACKET socket. This is fairly simple. We immediately go
3174  *	to 'closed' state and remove our protocol entry in the device list.
3175  */
3176 
packet_release(struct socket * sock)3177 static int packet_release(struct socket *sock)
3178 {
3179 	struct sock *sk = sock->sk;
3180 	struct packet_sock *po;
3181 	struct packet_fanout *f;
3182 	struct net *net;
3183 	union tpacket_req_u req_u;
3184 
3185 	if (!sk)
3186 		return 0;
3187 
3188 	net = sock_net(sk);
3189 	po = pkt_sk(sk);
3190 
3191 	mutex_lock(&net->packet.sklist_lock);
3192 	sk_del_node_init_rcu(sk);
3193 	mutex_unlock(&net->packet.sklist_lock);
3194 
3195 	sock_prot_inuse_add(net, sk->sk_prot, -1);
3196 
3197 	spin_lock(&po->bind_lock);
3198 	unregister_prot_hook(sk, false);
3199 	packet_cached_dev_reset(po);
3200 
3201 	if (po->prot_hook.dev) {
3202 		netdev_put(po->prot_hook.dev, &po->prot_hook.dev_tracker);
3203 		po->prot_hook.dev = NULL;
3204 	}
3205 	spin_unlock(&po->bind_lock);
3206 
3207 	packet_flush_mclist(sk);
3208 
3209 	lock_sock(sk);
3210 	if (po->rx_ring.pg_vec) {
3211 		memset(&req_u, 0, sizeof(req_u));
3212 		packet_set_ring(sk, &req_u, 1, 0);
3213 	}
3214 
3215 	if (po->tx_ring.pg_vec) {
3216 		memset(&req_u, 0, sizeof(req_u));
3217 		packet_set_ring(sk, &req_u, 1, 1);
3218 	}
3219 	release_sock(sk);
3220 
3221 	f = fanout_release(sk);
3222 
3223 	synchronize_net();
3224 
3225 	kfree(po->rollover);
3226 	if (f) {
3227 		fanout_release_data(f);
3228 		kvfree(f);
3229 	}
3230 	/*
3231 	 *	Now the socket is dead. No more input will appear.
3232 	 */
3233 	sock_orphan(sk);
3234 	sock->sk = NULL;
3235 
3236 	/* Purge queues */
3237 
3238 	skb_queue_purge(&sk->sk_receive_queue);
3239 	packet_free_pending(po);
3240 
3241 	sock_put(sk);
3242 	return 0;
3243 }
3244 
3245 /*
3246  *	Attach a packet hook.
3247  */
3248 
packet_do_bind(struct sock * sk,const char * name,int ifindex,__be16 proto)3249 static int packet_do_bind(struct sock *sk, const char *name, int ifindex,
3250 			  __be16 proto)
3251 {
3252 	struct packet_sock *po = pkt_sk(sk);
3253 	struct net_device *dev = NULL;
3254 	bool unlisted = false;
3255 	bool need_rehook;
3256 	int ret = 0;
3257 
3258 	lock_sock(sk);
3259 	spin_lock(&po->bind_lock);
3260 	if (!proto)
3261 		proto = po->num;
3262 
3263 	rcu_read_lock();
3264 
3265 	if (po->fanout) {
3266 		ret = -EINVAL;
3267 		goto out_unlock;
3268 	}
3269 
3270 	if (name) {
3271 		dev = dev_get_by_name_rcu(sock_net(sk), name);
3272 		if (!dev) {
3273 			ret = -ENODEV;
3274 			goto out_unlock;
3275 		}
3276 	} else if (ifindex) {
3277 		dev = dev_get_by_index_rcu(sock_net(sk), ifindex);
3278 		if (!dev) {
3279 			ret = -ENODEV;
3280 			goto out_unlock;
3281 		}
3282 	}
3283 
3284 	need_rehook = po->prot_hook.type != proto || po->prot_hook.dev != dev;
3285 
3286 	if (need_rehook) {
3287 		dev_hold(dev);
3288 		if (packet_sock_flag(po, PACKET_SOCK_RUNNING)) {
3289 			rcu_read_unlock();
3290 			/* prevents packet_notifier() from calling
3291 			 * register_prot_hook()
3292 			 */
3293 			WRITE_ONCE(po->num, 0);
3294 			__unregister_prot_hook(sk, true);
3295 			rcu_read_lock();
3296 			if (dev)
3297 				unlisted = !dev_get_by_index_rcu(sock_net(sk),
3298 								 dev->ifindex);
3299 		}
3300 
3301 		BUG_ON(packet_sock_flag(po, PACKET_SOCK_RUNNING));
3302 		WRITE_ONCE(po->num, proto);
3303 		po->prot_hook.type = proto;
3304 
3305 		netdev_put(po->prot_hook.dev, &po->prot_hook.dev_tracker);
3306 
3307 		if (unlikely(unlisted)) {
3308 			po->prot_hook.dev = NULL;
3309 			WRITE_ONCE(po->ifindex, -1);
3310 			packet_cached_dev_reset(po);
3311 		} else {
3312 			netdev_hold(dev, &po->prot_hook.dev_tracker,
3313 				    GFP_ATOMIC);
3314 			po->prot_hook.dev = dev;
3315 			WRITE_ONCE(po->ifindex, dev ? dev->ifindex : 0);
3316 			packet_cached_dev_assign(po, dev);
3317 		}
3318 		dev_put(dev);
3319 	}
3320 
3321 	if (proto == 0 || !need_rehook)
3322 		goto out_unlock;
3323 
3324 	if (!unlisted && (!dev || (dev->flags & IFF_UP))) {
3325 		register_prot_hook(sk);
3326 	} else {
3327 		sk->sk_err = ENETDOWN;
3328 		if (!sock_flag(sk, SOCK_DEAD))
3329 			sk_error_report(sk);
3330 	}
3331 
3332 out_unlock:
3333 	rcu_read_unlock();
3334 	spin_unlock(&po->bind_lock);
3335 	release_sock(sk);
3336 	return ret;
3337 }
3338 
3339 /*
3340  *	Bind a packet socket to a device
3341  */
3342 
packet_bind_spkt(struct socket * sock,struct sockaddr * uaddr,int addr_len)3343 static int packet_bind_spkt(struct socket *sock, struct sockaddr *uaddr,
3344 			    int addr_len)
3345 {
3346 	struct sock *sk = sock->sk;
3347 	char name[sizeof(uaddr->sa_data_min) + 1];
3348 
3349 	/*
3350 	 *	Check legality
3351 	 */
3352 
3353 	if (addr_len != sizeof(struct sockaddr))
3354 		return -EINVAL;
3355 	/* uaddr->sa_data comes from the userspace, it's not guaranteed to be
3356 	 * zero-terminated.
3357 	 */
3358 	memcpy(name, uaddr->sa_data, sizeof(uaddr->sa_data_min));
3359 	name[sizeof(uaddr->sa_data_min)] = 0;
3360 
3361 	return packet_do_bind(sk, name, 0, 0);
3362 }
3363 
packet_bind(struct socket * sock,struct sockaddr * uaddr,int addr_len)3364 static int packet_bind(struct socket *sock, struct sockaddr *uaddr, int addr_len)
3365 {
3366 	struct sockaddr_ll *sll = (struct sockaddr_ll *)uaddr;
3367 	struct sock *sk = sock->sk;
3368 
3369 	/*
3370 	 *	Check legality
3371 	 */
3372 
3373 	if (addr_len < sizeof(struct sockaddr_ll))
3374 		return -EINVAL;
3375 	if (sll->sll_family != AF_PACKET)
3376 		return -EINVAL;
3377 
3378 	return packet_do_bind(sk, NULL, sll->sll_ifindex, sll->sll_protocol);
3379 }
3380 
3381 static struct proto packet_proto = {
3382 	.name	  = "PACKET",
3383 	.owner	  = THIS_MODULE,
3384 	.obj_size = sizeof(struct packet_sock),
3385 };
3386 
3387 /*
3388  *	Create a packet of type SOCK_PACKET.
3389  */
3390 
packet_create(struct net * net,struct socket * sock,int protocol,int kern)3391 static int packet_create(struct net *net, struct socket *sock, int protocol,
3392 			 int kern)
3393 {
3394 	struct sock *sk;
3395 	struct packet_sock *po;
3396 	__be16 proto = (__force __be16)protocol; /* weird, but documented */
3397 	int err;
3398 
3399 	if (!ns_capable(net->user_ns, CAP_NET_RAW))
3400 		return -EPERM;
3401 	if (sock->type != SOCK_DGRAM && sock->type != SOCK_RAW &&
3402 	    sock->type != SOCK_PACKET)
3403 		return -ESOCKTNOSUPPORT;
3404 
3405 	sock->state = SS_UNCONNECTED;
3406 
3407 	err = -ENOBUFS;
3408 	sk = sk_alloc(net, PF_PACKET, GFP_KERNEL, &packet_proto, kern);
3409 	if (sk == NULL)
3410 		goto out;
3411 
3412 	sock->ops = &packet_ops;
3413 	if (sock->type == SOCK_PACKET)
3414 		sock->ops = &packet_ops_spkt;
3415 
3416 	po = pkt_sk(sk);
3417 	err = packet_alloc_pending(po);
3418 	if (err)
3419 		goto out_sk_free;
3420 
3421 	sock_init_data(sock, sk);
3422 
3423 	init_completion(&po->skb_completion);
3424 	sk->sk_family = PF_PACKET;
3425 	po->num = proto;
3426 
3427 	packet_cached_dev_reset(po);
3428 
3429 	sk->sk_destruct = packet_sock_destruct;
3430 
3431 	/*
3432 	 *	Attach a protocol block
3433 	 */
3434 
3435 	spin_lock_init(&po->bind_lock);
3436 	mutex_init(&po->pg_vec_lock);
3437 	po->rollover = NULL;
3438 	po->prot_hook.func = packet_rcv;
3439 
3440 	if (sock->type == SOCK_PACKET)
3441 		po->prot_hook.func = packet_rcv_spkt;
3442 
3443 	po->prot_hook.af_packet_priv = sk;
3444 	po->prot_hook.af_packet_net = sock_net(sk);
3445 
3446 	if (proto) {
3447 		po->prot_hook.type = proto;
3448 		__register_prot_hook(sk);
3449 	}
3450 
3451 	mutex_lock(&net->packet.sklist_lock);
3452 	sk_add_node_tail_rcu(sk, &net->packet.sklist);
3453 	mutex_unlock(&net->packet.sklist_lock);
3454 
3455 	sock_prot_inuse_add(net, &packet_proto, 1);
3456 
3457 	return 0;
3458 out_sk_free:
3459 	sk_free(sk);
3460 out:
3461 	return err;
3462 }
3463 
3464 /*
3465  *	Pull a packet from our receive queue and hand it to the user.
3466  *	If necessary we block.
3467  */
3468 
packet_recvmsg(struct socket * sock,struct msghdr * msg,size_t len,int flags)3469 static int packet_recvmsg(struct socket *sock, struct msghdr *msg, size_t len,
3470 			  int flags)
3471 {
3472 	struct sock *sk = sock->sk;
3473 	struct sk_buff *skb;
3474 	int copied, err;
3475 	int vnet_hdr_len = READ_ONCE(pkt_sk(sk)->vnet_hdr_sz);
3476 	unsigned int origlen = 0;
3477 
3478 	err = -EINVAL;
3479 	if (flags & ~(MSG_PEEK|MSG_DONTWAIT|MSG_TRUNC|MSG_CMSG_COMPAT|MSG_ERRQUEUE))
3480 		goto out;
3481 
3482 #if 0
3483 	/* What error should we return now? EUNATTACH? */
3484 	if (pkt_sk(sk)->ifindex < 0)
3485 		return -ENODEV;
3486 #endif
3487 
3488 	if (flags & MSG_ERRQUEUE) {
3489 		err = sock_recv_errqueue(sk, msg, len,
3490 					 SOL_PACKET, PACKET_TX_TIMESTAMP);
3491 		goto out;
3492 	}
3493 
3494 	/*
3495 	 *	Call the generic datagram receiver. This handles all sorts
3496 	 *	of horrible races and re-entrancy so we can forget about it
3497 	 *	in the protocol layers.
3498 	 *
3499 	 *	Now it will return ENETDOWN, if device have just gone down,
3500 	 *	but then it will block.
3501 	 */
3502 
3503 	skb = skb_recv_datagram(sk, flags, &err);
3504 
3505 	/*
3506 	 *	An error occurred so return it. Because skb_recv_datagram()
3507 	 *	handles the blocking we don't see and worry about blocking
3508 	 *	retries.
3509 	 */
3510 
3511 	if (skb == NULL)
3512 		goto out;
3513 
3514 	packet_rcv_try_clear_pressure(pkt_sk(sk));
3515 
3516 	if (vnet_hdr_len) {
3517 		err = packet_rcv_vnet(msg, skb, &len, vnet_hdr_len);
3518 		if (err)
3519 			goto out_free;
3520 	}
3521 
3522 	/* You lose any data beyond the buffer you gave. If it worries
3523 	 * a user program they can ask the device for its MTU
3524 	 * anyway.
3525 	 */
3526 	copied = skb->len;
3527 	if (copied > len) {
3528 		copied = len;
3529 		msg->msg_flags |= MSG_TRUNC;
3530 	}
3531 
3532 	err = skb_copy_datagram_msg(skb, 0, msg, copied);
3533 	if (err)
3534 		goto out_free;
3535 
3536 	if (sock->type != SOCK_PACKET) {
3537 		struct sockaddr_ll *sll = &PACKET_SKB_CB(skb)->sa.ll;
3538 
3539 		/* Original length was stored in sockaddr_ll fields */
3540 		origlen = PACKET_SKB_CB(skb)->sa.origlen;
3541 		sll->sll_family = AF_PACKET;
3542 		sll->sll_protocol = (sock->type == SOCK_DGRAM) ?
3543 			vlan_get_protocol_dgram(skb) : skb->protocol;
3544 	}
3545 
3546 	sock_recv_cmsgs(msg, sk, skb);
3547 
3548 	if (msg->msg_name) {
3549 		const size_t max_len = min(sizeof(skb->cb),
3550 					   sizeof(struct sockaddr_storage));
3551 		int copy_len;
3552 
3553 		/* If the address length field is there to be filled
3554 		 * in, we fill it in now.
3555 		 */
3556 		if (sock->type == SOCK_PACKET) {
3557 			__sockaddr_check_size(sizeof(struct sockaddr_pkt));
3558 			msg->msg_namelen = sizeof(struct sockaddr_pkt);
3559 			copy_len = msg->msg_namelen;
3560 		} else {
3561 			struct sockaddr_ll *sll = &PACKET_SKB_CB(skb)->sa.ll;
3562 
3563 			msg->msg_namelen = sll->sll_halen +
3564 				offsetof(struct sockaddr_ll, sll_addr);
3565 			copy_len = msg->msg_namelen;
3566 			if (msg->msg_namelen < sizeof(struct sockaddr_ll)) {
3567 				memset(msg->msg_name +
3568 				       offsetof(struct sockaddr_ll, sll_addr),
3569 				       0, sizeof(sll->sll_addr));
3570 				msg->msg_namelen = sizeof(struct sockaddr_ll);
3571 			}
3572 		}
3573 		if (WARN_ON_ONCE(copy_len > max_len)) {
3574 			copy_len = max_len;
3575 			msg->msg_namelen = copy_len;
3576 		}
3577 		memcpy(msg->msg_name, &PACKET_SKB_CB(skb)->sa, copy_len);
3578 	}
3579 
3580 	if (packet_sock_flag(pkt_sk(sk), PACKET_SOCK_AUXDATA)) {
3581 		struct tpacket_auxdata aux;
3582 
3583 		aux.tp_status = TP_STATUS_USER;
3584 		if (skb->ip_summed == CHECKSUM_PARTIAL)
3585 			aux.tp_status |= TP_STATUS_CSUMNOTREADY;
3586 		else if (skb->pkt_type != PACKET_OUTGOING &&
3587 			 skb_csum_unnecessary(skb))
3588 			aux.tp_status |= TP_STATUS_CSUM_VALID;
3589 		if (skb_is_gso(skb) && skb_is_gso_tcp(skb))
3590 			aux.tp_status |= TP_STATUS_GSO_TCP;
3591 
3592 		aux.tp_len = origlen;
3593 		aux.tp_snaplen = skb->len;
3594 		aux.tp_mac = 0;
3595 		aux.tp_net = skb_network_offset(skb);
3596 		if (skb_vlan_tag_present(skb)) {
3597 			aux.tp_vlan_tci = skb_vlan_tag_get(skb);
3598 			aux.tp_vlan_tpid = ntohs(skb->vlan_proto);
3599 			aux.tp_status |= TP_STATUS_VLAN_VALID | TP_STATUS_VLAN_TPID_VALID;
3600 		} else if (unlikely(sock->type == SOCK_DGRAM && eth_type_vlan(skb->protocol))) {
3601 			struct sockaddr_ll *sll = &PACKET_SKB_CB(skb)->sa.ll;
3602 			struct net_device *dev;
3603 
3604 			rcu_read_lock();
3605 			dev = dev_get_by_index_rcu(sock_net(sk), sll->sll_ifindex);
3606 			if (dev) {
3607 				aux.tp_vlan_tci = vlan_get_tci(skb, dev);
3608 				aux.tp_vlan_tpid = ntohs(skb->protocol);
3609 				aux.tp_status |= TP_STATUS_VLAN_VALID | TP_STATUS_VLAN_TPID_VALID;
3610 			} else {
3611 				aux.tp_vlan_tci = 0;
3612 				aux.tp_vlan_tpid = 0;
3613 			}
3614 			rcu_read_unlock();
3615 		} else {
3616 			aux.tp_vlan_tci = 0;
3617 			aux.tp_vlan_tpid = 0;
3618 		}
3619 		put_cmsg(msg, SOL_PACKET, PACKET_AUXDATA, sizeof(aux), &aux);
3620 	}
3621 
3622 	/*
3623 	 *	Free or return the buffer as appropriate. Again this
3624 	 *	hides all the races and re-entrancy issues from us.
3625 	 */
3626 	err = vnet_hdr_len + ((flags&MSG_TRUNC) ? skb->len : copied);
3627 
3628 out_free:
3629 	skb_free_datagram(sk, skb);
3630 out:
3631 	return err;
3632 }
3633 
packet_getname_spkt(struct socket * sock,struct sockaddr * uaddr,int peer)3634 static int packet_getname_spkt(struct socket *sock, struct sockaddr *uaddr,
3635 			       int peer)
3636 {
3637 	struct net_device *dev;
3638 	struct sock *sk	= sock->sk;
3639 
3640 	if (peer)
3641 		return -EOPNOTSUPP;
3642 
3643 	uaddr->sa_family = AF_PACKET;
3644 	memset(uaddr->sa_data, 0, sizeof(uaddr->sa_data_min));
3645 	rcu_read_lock();
3646 	dev = dev_get_by_index_rcu(sock_net(sk), READ_ONCE(pkt_sk(sk)->ifindex));
3647 	if (dev)
3648 		strscpy(uaddr->sa_data, dev->name, sizeof(uaddr->sa_data_min));
3649 	rcu_read_unlock();
3650 
3651 	return sizeof(*uaddr);
3652 }
3653 
packet_getname(struct socket * sock,struct sockaddr * uaddr,int peer)3654 static int packet_getname(struct socket *sock, struct sockaddr *uaddr,
3655 			  int peer)
3656 {
3657 	struct net_device *dev;
3658 	struct sock *sk = sock->sk;
3659 	struct packet_sock *po = pkt_sk(sk);
3660 	DECLARE_SOCKADDR(struct sockaddr_ll *, sll, uaddr);
3661 	int ifindex;
3662 
3663 	if (peer)
3664 		return -EOPNOTSUPP;
3665 
3666 	ifindex = READ_ONCE(po->ifindex);
3667 	sll->sll_family = AF_PACKET;
3668 	sll->sll_ifindex = ifindex;
3669 	sll->sll_protocol = READ_ONCE(po->num);
3670 	sll->sll_pkttype = 0;
3671 	rcu_read_lock();
3672 	dev = dev_get_by_index_rcu(sock_net(sk), ifindex);
3673 	if (dev) {
3674 		sll->sll_hatype = dev->type;
3675 		sll->sll_halen = dev->addr_len;
3676 
3677 		/* Let __fortify_memcpy_chk() know the actual buffer size. */
3678 		memcpy(((struct sockaddr_storage *)sll)->__data +
3679 		       offsetof(struct sockaddr_ll, sll_addr) -
3680 		       offsetofend(struct sockaddr_ll, sll_family),
3681 		       dev->dev_addr, dev->addr_len);
3682 	} else {
3683 		sll->sll_hatype = 0;	/* Bad: we have no ARPHRD_UNSPEC */
3684 		sll->sll_halen = 0;
3685 	}
3686 	rcu_read_unlock();
3687 
3688 	return offsetof(struct sockaddr_ll, sll_addr) + sll->sll_halen;
3689 }
3690 
packet_dev_mc(struct net_device * dev,struct packet_mclist * i,int what)3691 static int packet_dev_mc(struct net_device *dev, struct packet_mclist *i,
3692 			 int what)
3693 {
3694 	switch (i->type) {
3695 	case PACKET_MR_MULTICAST:
3696 		if (i->alen != dev->addr_len)
3697 			return -EINVAL;
3698 		if (what > 0)
3699 			return dev_mc_add(dev, i->addr);
3700 		else
3701 			return dev_mc_del(dev, i->addr);
3702 		break;
3703 	case PACKET_MR_PROMISC:
3704 		return dev_set_promiscuity(dev, what);
3705 	case PACKET_MR_ALLMULTI:
3706 		return dev_set_allmulti(dev, what);
3707 	case PACKET_MR_UNICAST:
3708 		if (i->alen != dev->addr_len)
3709 			return -EINVAL;
3710 		if (what > 0)
3711 			return dev_uc_add(dev, i->addr);
3712 		else
3713 			return dev_uc_del(dev, i->addr);
3714 		break;
3715 	default:
3716 		break;
3717 	}
3718 	return 0;
3719 }
3720 
packet_dev_mclist_delete(struct net_device * dev,struct packet_mclist ** mlp)3721 static void packet_dev_mclist_delete(struct net_device *dev,
3722 				     struct packet_mclist **mlp)
3723 {
3724 	struct packet_mclist *ml;
3725 
3726 	while ((ml = *mlp) != NULL) {
3727 		if (ml->ifindex == dev->ifindex) {
3728 			packet_dev_mc(dev, ml, -1);
3729 			*mlp = ml->next;
3730 			kfree(ml);
3731 		} else
3732 			mlp = &ml->next;
3733 	}
3734 }
3735 
packet_mc_add(struct sock * sk,struct packet_mreq_max * mreq)3736 static int packet_mc_add(struct sock *sk, struct packet_mreq_max *mreq)
3737 {
3738 	struct packet_sock *po = pkt_sk(sk);
3739 	struct packet_mclist *ml, *i;
3740 	struct net_device *dev;
3741 	int err;
3742 
3743 	rtnl_lock();
3744 
3745 	err = -ENODEV;
3746 	dev = __dev_get_by_index(sock_net(sk), mreq->mr_ifindex);
3747 	if (!dev)
3748 		goto done;
3749 
3750 	err = -EINVAL;
3751 	if (mreq->mr_alen > dev->addr_len)
3752 		goto done;
3753 
3754 	err = -ENOBUFS;
3755 	i = kmalloc(sizeof(*i), GFP_KERNEL);
3756 	if (i == NULL)
3757 		goto done;
3758 
3759 	err = 0;
3760 	for (ml = po->mclist; ml; ml = ml->next) {
3761 		if (ml->ifindex == mreq->mr_ifindex &&
3762 		    ml->type == mreq->mr_type &&
3763 		    ml->alen == mreq->mr_alen &&
3764 		    memcmp(ml->addr, mreq->mr_address, ml->alen) == 0) {
3765 			ml->count++;
3766 			/* Free the new element ... */
3767 			kfree(i);
3768 			goto done;
3769 		}
3770 	}
3771 
3772 	i->type = mreq->mr_type;
3773 	i->ifindex = mreq->mr_ifindex;
3774 	i->alen = mreq->mr_alen;
3775 	memcpy(i->addr, mreq->mr_address, i->alen);
3776 	memset(i->addr + i->alen, 0, sizeof(i->addr) - i->alen);
3777 	i->count = 1;
3778 	i->next = po->mclist;
3779 	po->mclist = i;
3780 	err = packet_dev_mc(dev, i, 1);
3781 	if (err) {
3782 		po->mclist = i->next;
3783 		kfree(i);
3784 	}
3785 
3786 done:
3787 	rtnl_unlock();
3788 	return err;
3789 }
3790 
packet_mc_drop(struct sock * sk,struct packet_mreq_max * mreq)3791 static int packet_mc_drop(struct sock *sk, struct packet_mreq_max *mreq)
3792 {
3793 	struct packet_mclist *ml, **mlp;
3794 
3795 	rtnl_lock();
3796 
3797 	for (mlp = &pkt_sk(sk)->mclist; (ml = *mlp) != NULL; mlp = &ml->next) {
3798 		if (ml->ifindex == mreq->mr_ifindex &&
3799 		    ml->type == mreq->mr_type &&
3800 		    ml->alen == mreq->mr_alen &&
3801 		    memcmp(ml->addr, mreq->mr_address, ml->alen) == 0) {
3802 			if (--ml->count == 0) {
3803 				struct net_device *dev;
3804 				*mlp = ml->next;
3805 				dev = __dev_get_by_index(sock_net(sk), ml->ifindex);
3806 				if (dev)
3807 					packet_dev_mc(dev, ml, -1);
3808 				kfree(ml);
3809 			}
3810 			break;
3811 		}
3812 	}
3813 	rtnl_unlock();
3814 	return 0;
3815 }
3816 
packet_flush_mclist(struct sock * sk)3817 static void packet_flush_mclist(struct sock *sk)
3818 {
3819 	struct packet_sock *po = pkt_sk(sk);
3820 	struct packet_mclist *ml;
3821 
3822 	if (!po->mclist)
3823 		return;
3824 
3825 	rtnl_lock();
3826 	while ((ml = po->mclist) != NULL) {
3827 		struct net_device *dev;
3828 
3829 		po->mclist = ml->next;
3830 		dev = __dev_get_by_index(sock_net(sk), ml->ifindex);
3831 		if (dev != NULL)
3832 			packet_dev_mc(dev, ml, -1);
3833 		kfree(ml);
3834 	}
3835 	rtnl_unlock();
3836 }
3837 
3838 static int
packet_setsockopt(struct socket * sock,int level,int optname,sockptr_t optval,unsigned int optlen)3839 packet_setsockopt(struct socket *sock, int level, int optname, sockptr_t optval,
3840 		  unsigned int optlen)
3841 {
3842 	struct sock *sk = sock->sk;
3843 	struct packet_sock *po = pkt_sk(sk);
3844 	int ret;
3845 
3846 	if (level != SOL_PACKET)
3847 		return -ENOPROTOOPT;
3848 
3849 	switch (optname) {
3850 	case PACKET_ADD_MEMBERSHIP:
3851 	case PACKET_DROP_MEMBERSHIP:
3852 	{
3853 		struct packet_mreq_max mreq;
3854 		int len = optlen;
3855 		memset(&mreq, 0, sizeof(mreq));
3856 		if (len < sizeof(struct packet_mreq))
3857 			return -EINVAL;
3858 		if (len > sizeof(mreq))
3859 			len = sizeof(mreq);
3860 		if (copy_from_sockptr(&mreq, optval, len))
3861 			return -EFAULT;
3862 		if (len < (mreq.mr_alen + offsetof(struct packet_mreq, mr_address)))
3863 			return -EINVAL;
3864 		if (optname == PACKET_ADD_MEMBERSHIP)
3865 			ret = packet_mc_add(sk, &mreq);
3866 		else
3867 			ret = packet_mc_drop(sk, &mreq);
3868 		return ret;
3869 	}
3870 
3871 	case PACKET_RX_RING:
3872 	case PACKET_TX_RING:
3873 	{
3874 		union tpacket_req_u req_u;
3875 
3876 		ret = -EINVAL;
3877 		lock_sock(sk);
3878 		switch (po->tp_version) {
3879 		case TPACKET_V1:
3880 		case TPACKET_V2:
3881 			if (optlen < sizeof(req_u.req))
3882 				break;
3883 			ret = copy_from_sockptr(&req_u.req, optval,
3884 						sizeof(req_u.req)) ?
3885 						-EINVAL : 0;
3886 			break;
3887 		case TPACKET_V3:
3888 		default:
3889 			if (optlen < sizeof(req_u.req3))
3890 				break;
3891 			ret = copy_from_sockptr(&req_u.req3, optval,
3892 						sizeof(req_u.req3)) ?
3893 						-EINVAL : 0;
3894 			break;
3895 		}
3896 		if (!ret)
3897 			ret = packet_set_ring(sk, &req_u, 0,
3898 					      optname == PACKET_TX_RING);
3899 		release_sock(sk);
3900 		return ret;
3901 	}
3902 	case PACKET_COPY_THRESH:
3903 	{
3904 		int val;
3905 
3906 		if (optlen != sizeof(val))
3907 			return -EINVAL;
3908 		if (copy_from_sockptr(&val, optval, sizeof(val)))
3909 			return -EFAULT;
3910 
3911 		pkt_sk(sk)->copy_thresh = val;
3912 		return 0;
3913 	}
3914 	case PACKET_VERSION:
3915 	{
3916 		int val;
3917 
3918 		if (optlen != sizeof(val))
3919 			return -EINVAL;
3920 		if (copy_from_sockptr(&val, optval, sizeof(val)))
3921 			return -EFAULT;
3922 		switch (val) {
3923 		case TPACKET_V1:
3924 		case TPACKET_V2:
3925 		case TPACKET_V3:
3926 			break;
3927 		default:
3928 			return -EINVAL;
3929 		}
3930 		lock_sock(sk);
3931 		if (po->rx_ring.pg_vec || po->tx_ring.pg_vec) {
3932 			ret = -EBUSY;
3933 		} else {
3934 			po->tp_version = val;
3935 			ret = 0;
3936 		}
3937 		release_sock(sk);
3938 		return ret;
3939 	}
3940 	case PACKET_RESERVE:
3941 	{
3942 		unsigned int val;
3943 
3944 		if (optlen != sizeof(val))
3945 			return -EINVAL;
3946 		if (copy_from_sockptr(&val, optval, sizeof(val)))
3947 			return -EFAULT;
3948 		if (val > INT_MAX)
3949 			return -EINVAL;
3950 		lock_sock(sk);
3951 		if (po->rx_ring.pg_vec || po->tx_ring.pg_vec) {
3952 			ret = -EBUSY;
3953 		} else {
3954 			po->tp_reserve = val;
3955 			ret = 0;
3956 		}
3957 		release_sock(sk);
3958 		return ret;
3959 	}
3960 	case PACKET_LOSS:
3961 	{
3962 		unsigned int val;
3963 
3964 		if (optlen != sizeof(val))
3965 			return -EINVAL;
3966 		if (copy_from_sockptr(&val, optval, sizeof(val)))
3967 			return -EFAULT;
3968 
3969 		lock_sock(sk);
3970 		if (po->rx_ring.pg_vec || po->tx_ring.pg_vec) {
3971 			ret = -EBUSY;
3972 		} else {
3973 			packet_sock_flag_set(po, PACKET_SOCK_TP_LOSS, val);
3974 			ret = 0;
3975 		}
3976 		release_sock(sk);
3977 		return ret;
3978 	}
3979 	case PACKET_AUXDATA:
3980 	{
3981 		int val;
3982 
3983 		if (optlen < sizeof(val))
3984 			return -EINVAL;
3985 		if (copy_from_sockptr(&val, optval, sizeof(val)))
3986 			return -EFAULT;
3987 
3988 		packet_sock_flag_set(po, PACKET_SOCK_AUXDATA, val);
3989 		return 0;
3990 	}
3991 	case PACKET_ORIGDEV:
3992 	{
3993 		int val;
3994 
3995 		if (optlen < sizeof(val))
3996 			return -EINVAL;
3997 		if (copy_from_sockptr(&val, optval, sizeof(val)))
3998 			return -EFAULT;
3999 
4000 		packet_sock_flag_set(po, PACKET_SOCK_ORIGDEV, val);
4001 		return 0;
4002 	}
4003 	case PACKET_VNET_HDR:
4004 	case PACKET_VNET_HDR_SZ:
4005 	{
4006 		int val, hdr_len;
4007 
4008 		if (sock->type != SOCK_RAW)
4009 			return -EINVAL;
4010 		if (optlen < sizeof(val))
4011 			return -EINVAL;
4012 		if (copy_from_sockptr(&val, optval, sizeof(val)))
4013 			return -EFAULT;
4014 
4015 		if (optname == PACKET_VNET_HDR_SZ) {
4016 			if (val && val != sizeof(struct virtio_net_hdr) &&
4017 			    val != sizeof(struct virtio_net_hdr_mrg_rxbuf))
4018 				return -EINVAL;
4019 			hdr_len = val;
4020 		} else {
4021 			hdr_len = val ? sizeof(struct virtio_net_hdr) : 0;
4022 		}
4023 		lock_sock(sk);
4024 		if (po->rx_ring.pg_vec || po->tx_ring.pg_vec) {
4025 			ret = -EBUSY;
4026 		} else {
4027 			WRITE_ONCE(po->vnet_hdr_sz, hdr_len);
4028 			ret = 0;
4029 		}
4030 		release_sock(sk);
4031 		return ret;
4032 	}
4033 	case PACKET_TIMESTAMP:
4034 	{
4035 		int val;
4036 
4037 		if (optlen != sizeof(val))
4038 			return -EINVAL;
4039 		if (copy_from_sockptr(&val, optval, sizeof(val)))
4040 			return -EFAULT;
4041 
4042 		WRITE_ONCE(po->tp_tstamp, val);
4043 		return 0;
4044 	}
4045 	case PACKET_FANOUT:
4046 	{
4047 		struct fanout_args args = { 0 };
4048 
4049 		if (optlen != sizeof(int) && optlen != sizeof(args))
4050 			return -EINVAL;
4051 		if (copy_from_sockptr(&args, optval, optlen))
4052 			return -EFAULT;
4053 
4054 		return fanout_add(sk, &args);
4055 	}
4056 	case PACKET_FANOUT_DATA:
4057 	{
4058 		/* Paired with the WRITE_ONCE() in fanout_add() */
4059 		if (!READ_ONCE(po->fanout))
4060 			return -EINVAL;
4061 
4062 		return fanout_set_data(po, optval, optlen);
4063 	}
4064 	case PACKET_IGNORE_OUTGOING:
4065 	{
4066 		int val;
4067 
4068 		if (optlen != sizeof(val))
4069 			return -EINVAL;
4070 		if (copy_from_sockptr(&val, optval, sizeof(val)))
4071 			return -EFAULT;
4072 		if (val < 0 || val > 1)
4073 			return -EINVAL;
4074 
4075 		WRITE_ONCE(po->prot_hook.ignore_outgoing, !!val);
4076 		return 0;
4077 	}
4078 	case PACKET_TX_HAS_OFF:
4079 	{
4080 		unsigned int val;
4081 
4082 		if (optlen != sizeof(val))
4083 			return -EINVAL;
4084 		if (copy_from_sockptr(&val, optval, sizeof(val)))
4085 			return -EFAULT;
4086 
4087 		lock_sock(sk);
4088 		if (!po->rx_ring.pg_vec && !po->tx_ring.pg_vec)
4089 			packet_sock_flag_set(po, PACKET_SOCK_TX_HAS_OFF, val);
4090 
4091 		release_sock(sk);
4092 		return 0;
4093 	}
4094 	case PACKET_QDISC_BYPASS:
4095 	{
4096 		int val;
4097 
4098 		if (optlen != sizeof(val))
4099 			return -EINVAL;
4100 		if (copy_from_sockptr(&val, optval, sizeof(val)))
4101 			return -EFAULT;
4102 
4103 		packet_sock_flag_set(po, PACKET_SOCK_QDISC_BYPASS, val);
4104 		return 0;
4105 	}
4106 	default:
4107 		return -ENOPROTOOPT;
4108 	}
4109 }
4110 
packet_getsockopt(struct socket * sock,int level,int optname,char __user * optval,int __user * optlen)4111 static int packet_getsockopt(struct socket *sock, int level, int optname,
4112 			     char __user *optval, int __user *optlen)
4113 {
4114 	int len;
4115 	int val, lv = sizeof(val);
4116 	struct sock *sk = sock->sk;
4117 	struct packet_sock *po = pkt_sk(sk);
4118 	void *data = &val;
4119 	union tpacket_stats_u st;
4120 	struct tpacket_rollover_stats rstats;
4121 	int drops;
4122 
4123 	if (level != SOL_PACKET)
4124 		return -ENOPROTOOPT;
4125 
4126 	if (get_user(len, optlen))
4127 		return -EFAULT;
4128 
4129 	if (len < 0)
4130 		return -EINVAL;
4131 
4132 	switch (optname) {
4133 	case PACKET_STATISTICS:
4134 		spin_lock_bh(&sk->sk_receive_queue.lock);
4135 		memcpy(&st, &po->stats, sizeof(st));
4136 		memset(&po->stats, 0, sizeof(po->stats));
4137 		spin_unlock_bh(&sk->sk_receive_queue.lock);
4138 		drops = atomic_xchg(&po->tp_drops, 0);
4139 
4140 		if (po->tp_version == TPACKET_V3) {
4141 			lv = sizeof(struct tpacket_stats_v3);
4142 			st.stats3.tp_drops = drops;
4143 			st.stats3.tp_packets += drops;
4144 			data = &st.stats3;
4145 		} else {
4146 			lv = sizeof(struct tpacket_stats);
4147 			st.stats1.tp_drops = drops;
4148 			st.stats1.tp_packets += drops;
4149 			data = &st.stats1;
4150 		}
4151 
4152 		break;
4153 	case PACKET_AUXDATA:
4154 		val = packet_sock_flag(po, PACKET_SOCK_AUXDATA);
4155 		break;
4156 	case PACKET_ORIGDEV:
4157 		val = packet_sock_flag(po, PACKET_SOCK_ORIGDEV);
4158 		break;
4159 	case PACKET_VNET_HDR:
4160 		val = !!READ_ONCE(po->vnet_hdr_sz);
4161 		break;
4162 	case PACKET_VNET_HDR_SZ:
4163 		val = READ_ONCE(po->vnet_hdr_sz);
4164 		break;
4165 	case PACKET_VERSION:
4166 		val = po->tp_version;
4167 		break;
4168 	case PACKET_HDRLEN:
4169 		if (len > sizeof(int))
4170 			len = sizeof(int);
4171 		if (len < sizeof(int))
4172 			return -EINVAL;
4173 		if (copy_from_user(&val, optval, len))
4174 			return -EFAULT;
4175 		switch (val) {
4176 		case TPACKET_V1:
4177 			val = sizeof(struct tpacket_hdr);
4178 			break;
4179 		case TPACKET_V2:
4180 			val = sizeof(struct tpacket2_hdr);
4181 			break;
4182 		case TPACKET_V3:
4183 			val = sizeof(struct tpacket3_hdr);
4184 			break;
4185 		default:
4186 			return -EINVAL;
4187 		}
4188 		break;
4189 	case PACKET_RESERVE:
4190 		val = po->tp_reserve;
4191 		break;
4192 	case PACKET_LOSS:
4193 		val = packet_sock_flag(po, PACKET_SOCK_TP_LOSS);
4194 		break;
4195 	case PACKET_TIMESTAMP:
4196 		val = READ_ONCE(po->tp_tstamp);
4197 		break;
4198 	case PACKET_FANOUT:
4199 		val = (po->fanout ?
4200 		       ((u32)po->fanout->id |
4201 			((u32)po->fanout->type << 16) |
4202 			((u32)po->fanout->flags << 24)) :
4203 		       0);
4204 		break;
4205 	case PACKET_IGNORE_OUTGOING:
4206 		val = READ_ONCE(po->prot_hook.ignore_outgoing);
4207 		break;
4208 	case PACKET_ROLLOVER_STATS:
4209 		if (!po->rollover)
4210 			return -EINVAL;
4211 		rstats.tp_all = atomic_long_read(&po->rollover->num);
4212 		rstats.tp_huge = atomic_long_read(&po->rollover->num_huge);
4213 		rstats.tp_failed = atomic_long_read(&po->rollover->num_failed);
4214 		data = &rstats;
4215 		lv = sizeof(rstats);
4216 		break;
4217 	case PACKET_TX_HAS_OFF:
4218 		val = packet_sock_flag(po, PACKET_SOCK_TX_HAS_OFF);
4219 		break;
4220 	case PACKET_QDISC_BYPASS:
4221 		val = packet_sock_flag(po, PACKET_SOCK_QDISC_BYPASS);
4222 		break;
4223 	default:
4224 		return -ENOPROTOOPT;
4225 	}
4226 
4227 	if (len > lv)
4228 		len = lv;
4229 	if (put_user(len, optlen))
4230 		return -EFAULT;
4231 	if (copy_to_user(optval, data, len))
4232 		return -EFAULT;
4233 	return 0;
4234 }
4235 
packet_notifier(struct notifier_block * this,unsigned long msg,void * ptr)4236 static int packet_notifier(struct notifier_block *this,
4237 			   unsigned long msg, void *ptr)
4238 {
4239 	struct sock *sk;
4240 	struct net_device *dev = netdev_notifier_info_to_dev(ptr);
4241 	struct net *net = dev_net(dev);
4242 
4243 	rcu_read_lock();
4244 	sk_for_each_rcu(sk, &net->packet.sklist) {
4245 		struct packet_sock *po = pkt_sk(sk);
4246 
4247 		switch (msg) {
4248 		case NETDEV_UNREGISTER:
4249 			if (po->mclist)
4250 				packet_dev_mclist_delete(dev, &po->mclist);
4251 			fallthrough;
4252 
4253 		case NETDEV_DOWN:
4254 			if (dev->ifindex == po->ifindex) {
4255 				spin_lock(&po->bind_lock);
4256 				if (packet_sock_flag(po, PACKET_SOCK_RUNNING)) {
4257 					__unregister_prot_hook(sk, false);
4258 					sk->sk_err = ENETDOWN;
4259 					if (!sock_flag(sk, SOCK_DEAD))
4260 						sk_error_report(sk);
4261 				}
4262 				if (msg == NETDEV_UNREGISTER) {
4263 					packet_cached_dev_reset(po);
4264 					WRITE_ONCE(po->ifindex, -1);
4265 					netdev_put(po->prot_hook.dev,
4266 						   &po->prot_hook.dev_tracker);
4267 					po->prot_hook.dev = NULL;
4268 				}
4269 				spin_unlock(&po->bind_lock);
4270 			}
4271 			break;
4272 		case NETDEV_UP:
4273 			if (dev->ifindex == po->ifindex) {
4274 				spin_lock(&po->bind_lock);
4275 				if (po->num)
4276 					register_prot_hook(sk);
4277 				spin_unlock(&po->bind_lock);
4278 			}
4279 			break;
4280 		}
4281 	}
4282 	rcu_read_unlock();
4283 	return NOTIFY_DONE;
4284 }
4285 
4286 
packet_ioctl(struct socket * sock,unsigned int cmd,unsigned long arg)4287 static int packet_ioctl(struct socket *sock, unsigned int cmd,
4288 			unsigned long arg)
4289 {
4290 	struct sock *sk = sock->sk;
4291 
4292 	switch (cmd) {
4293 	case SIOCOUTQ:
4294 	{
4295 		int amount = sk_wmem_alloc_get(sk);
4296 
4297 		return put_user(amount, (int __user *)arg);
4298 	}
4299 	case SIOCINQ:
4300 	{
4301 		struct sk_buff *skb;
4302 		int amount = 0;
4303 
4304 		spin_lock_bh(&sk->sk_receive_queue.lock);
4305 		skb = skb_peek(&sk->sk_receive_queue);
4306 		if (skb)
4307 			amount = skb->len;
4308 		spin_unlock_bh(&sk->sk_receive_queue.lock);
4309 		return put_user(amount, (int __user *)arg);
4310 	}
4311 #ifdef CONFIG_INET
4312 	case SIOCADDRT:
4313 	case SIOCDELRT:
4314 	case SIOCDARP:
4315 	case SIOCGARP:
4316 	case SIOCSARP:
4317 	case SIOCGIFADDR:
4318 	case SIOCSIFADDR:
4319 	case SIOCGIFBRDADDR:
4320 	case SIOCSIFBRDADDR:
4321 	case SIOCGIFNETMASK:
4322 	case SIOCSIFNETMASK:
4323 	case SIOCGIFDSTADDR:
4324 	case SIOCSIFDSTADDR:
4325 	case SIOCSIFFLAGS:
4326 		return inet_dgram_ops.ioctl(sock, cmd, arg);
4327 #endif
4328 
4329 	default:
4330 		return -ENOIOCTLCMD;
4331 	}
4332 	return 0;
4333 }
4334 
packet_poll(struct file * file,struct socket * sock,poll_table * wait)4335 static __poll_t packet_poll(struct file *file, struct socket *sock,
4336 				poll_table *wait)
4337 {
4338 	struct sock *sk = sock->sk;
4339 	struct packet_sock *po = pkt_sk(sk);
4340 	__poll_t mask = datagram_poll(file, sock, wait);
4341 
4342 	spin_lock_bh(&sk->sk_receive_queue.lock);
4343 	if (po->rx_ring.pg_vec) {
4344 		if (!packet_previous_rx_frame(po, &po->rx_ring,
4345 			TP_STATUS_KERNEL))
4346 			mask |= EPOLLIN | EPOLLRDNORM;
4347 	}
4348 	packet_rcv_try_clear_pressure(po);
4349 	spin_unlock_bh(&sk->sk_receive_queue.lock);
4350 	spin_lock_bh(&sk->sk_write_queue.lock);
4351 	if (po->tx_ring.pg_vec) {
4352 		if (packet_current_frame(po, &po->tx_ring, TP_STATUS_AVAILABLE))
4353 			mask |= EPOLLOUT | EPOLLWRNORM;
4354 	}
4355 	spin_unlock_bh(&sk->sk_write_queue.lock);
4356 	return mask;
4357 }
4358 
4359 
4360 /* Dirty? Well, I still did not learn better way to account
4361  * for user mmaps.
4362  */
4363 
packet_mm_open(struct vm_area_struct * vma)4364 static void packet_mm_open(struct vm_area_struct *vma)
4365 {
4366 	struct file *file = vma->vm_file;
4367 	struct socket *sock = file->private_data;
4368 	struct sock *sk = sock->sk;
4369 
4370 	if (sk)
4371 		atomic_long_inc(&pkt_sk(sk)->mapped);
4372 }
4373 
packet_mm_close(struct vm_area_struct * vma)4374 static void packet_mm_close(struct vm_area_struct *vma)
4375 {
4376 	struct file *file = vma->vm_file;
4377 	struct socket *sock = file->private_data;
4378 	struct sock *sk = sock->sk;
4379 
4380 	if (sk)
4381 		atomic_long_dec(&pkt_sk(sk)->mapped);
4382 }
4383 
4384 static const struct vm_operations_struct packet_mmap_ops = {
4385 	.open	=	packet_mm_open,
4386 	.close	=	packet_mm_close,
4387 };
4388 
free_pg_vec(struct pgv * pg_vec,unsigned int order,unsigned int len)4389 static void free_pg_vec(struct pgv *pg_vec, unsigned int order,
4390 			unsigned int len)
4391 {
4392 	int i;
4393 
4394 	for (i = 0; i < len; i++) {
4395 		if (likely(pg_vec[i].buffer)) {
4396 			if (is_vmalloc_addr(pg_vec[i].buffer))
4397 				vfree(pg_vec[i].buffer);
4398 			else
4399 				free_pages((unsigned long)pg_vec[i].buffer,
4400 					   order);
4401 			pg_vec[i].buffer = NULL;
4402 		}
4403 	}
4404 	kfree(pg_vec);
4405 }
4406 
alloc_one_pg_vec_page(unsigned long order)4407 static char *alloc_one_pg_vec_page(unsigned long order)
4408 {
4409 	char *buffer;
4410 	gfp_t gfp_flags = GFP_KERNEL | __GFP_COMP |
4411 			  __GFP_ZERO | __GFP_NOWARN | __GFP_NORETRY;
4412 
4413 	buffer = (char *) __get_free_pages(gfp_flags, order);
4414 	if (buffer)
4415 		return buffer;
4416 
4417 	/* __get_free_pages failed, fall back to vmalloc */
4418 	buffer = vzalloc(array_size((1 << order), PAGE_SIZE));
4419 	if (buffer)
4420 		return buffer;
4421 
4422 	/* vmalloc failed, lets dig into swap here */
4423 	gfp_flags &= ~__GFP_NORETRY;
4424 	buffer = (char *) __get_free_pages(gfp_flags, order);
4425 	if (buffer)
4426 		return buffer;
4427 
4428 	/* complete and utter failure */
4429 	return NULL;
4430 }
4431 
alloc_pg_vec(struct tpacket_req * req,int order)4432 static struct pgv *alloc_pg_vec(struct tpacket_req *req, int order)
4433 {
4434 	unsigned int block_nr = req->tp_block_nr;
4435 	struct pgv *pg_vec;
4436 	int i;
4437 
4438 	pg_vec = kcalloc(block_nr, sizeof(struct pgv), GFP_KERNEL | __GFP_NOWARN);
4439 	if (unlikely(!pg_vec))
4440 		goto out;
4441 
4442 	for (i = 0; i < block_nr; i++) {
4443 		pg_vec[i].buffer = alloc_one_pg_vec_page(order);
4444 		if (unlikely(!pg_vec[i].buffer))
4445 			goto out_free_pgvec;
4446 	}
4447 
4448 out:
4449 	return pg_vec;
4450 
4451 out_free_pgvec:
4452 	free_pg_vec(pg_vec, order, block_nr);
4453 	pg_vec = NULL;
4454 	goto out;
4455 }
4456 
packet_set_ring(struct sock * sk,union tpacket_req_u * req_u,int closing,int tx_ring)4457 static int packet_set_ring(struct sock *sk, union tpacket_req_u *req_u,
4458 		int closing, int tx_ring)
4459 {
4460 	struct pgv *pg_vec = NULL;
4461 	struct packet_sock *po = pkt_sk(sk);
4462 	unsigned long *rx_owner_map = NULL;
4463 	int was_running, order = 0;
4464 	struct packet_ring_buffer *rb;
4465 	struct sk_buff_head *rb_queue;
4466 	__be16 num;
4467 	int err;
4468 	/* Added to avoid minimal code churn */
4469 	struct tpacket_req *req = &req_u->req;
4470 
4471 	rb = tx_ring ? &po->tx_ring : &po->rx_ring;
4472 	rb_queue = tx_ring ? &sk->sk_write_queue : &sk->sk_receive_queue;
4473 
4474 	err = -EBUSY;
4475 	if (!closing) {
4476 		if (atomic_long_read(&po->mapped))
4477 			goto out;
4478 		if (packet_read_pending(rb))
4479 			goto out;
4480 	}
4481 
4482 	if (req->tp_block_nr) {
4483 		unsigned int min_frame_size;
4484 
4485 		/* Sanity tests and some calculations */
4486 		err = -EBUSY;
4487 		if (unlikely(rb->pg_vec))
4488 			goto out;
4489 
4490 		switch (po->tp_version) {
4491 		case TPACKET_V1:
4492 			po->tp_hdrlen = TPACKET_HDRLEN;
4493 			break;
4494 		case TPACKET_V2:
4495 			po->tp_hdrlen = TPACKET2_HDRLEN;
4496 			break;
4497 		case TPACKET_V3:
4498 			po->tp_hdrlen = TPACKET3_HDRLEN;
4499 			break;
4500 		}
4501 
4502 		err = -EINVAL;
4503 		if (unlikely((int)req->tp_block_size <= 0))
4504 			goto out;
4505 		if (unlikely(!PAGE_ALIGNED(req->tp_block_size)))
4506 			goto out;
4507 		min_frame_size = po->tp_hdrlen + po->tp_reserve;
4508 		if (po->tp_version >= TPACKET_V3 &&
4509 		    req->tp_block_size <
4510 		    BLK_PLUS_PRIV((u64)req_u->req3.tp_sizeof_priv) + min_frame_size)
4511 			goto out;
4512 		if (unlikely(req->tp_frame_size < min_frame_size))
4513 			goto out;
4514 		if (unlikely(req->tp_frame_size & (TPACKET_ALIGNMENT - 1)))
4515 			goto out;
4516 
4517 		rb->frames_per_block = req->tp_block_size / req->tp_frame_size;
4518 		if (unlikely(rb->frames_per_block == 0))
4519 			goto out;
4520 		if (unlikely(rb->frames_per_block > UINT_MAX / req->tp_block_nr))
4521 			goto out;
4522 		if (unlikely((rb->frames_per_block * req->tp_block_nr) !=
4523 					req->tp_frame_nr))
4524 			goto out;
4525 
4526 		err = -ENOMEM;
4527 		order = get_order(req->tp_block_size);
4528 		pg_vec = alloc_pg_vec(req, order);
4529 		if (unlikely(!pg_vec))
4530 			goto out;
4531 		switch (po->tp_version) {
4532 		case TPACKET_V3:
4533 			/* Block transmit is not supported yet */
4534 			if (!tx_ring) {
4535 				init_prb_bdqc(po, rb, pg_vec, req_u);
4536 			} else {
4537 				struct tpacket_req3 *req3 = &req_u->req3;
4538 
4539 				if (req3->tp_retire_blk_tov ||
4540 				    req3->tp_sizeof_priv ||
4541 				    req3->tp_feature_req_word) {
4542 					err = -EINVAL;
4543 					goto out_free_pg_vec;
4544 				}
4545 			}
4546 			break;
4547 		default:
4548 			if (!tx_ring) {
4549 				rx_owner_map = bitmap_alloc(req->tp_frame_nr,
4550 					GFP_KERNEL | __GFP_NOWARN | __GFP_ZERO);
4551 				if (!rx_owner_map)
4552 					goto out_free_pg_vec;
4553 			}
4554 			break;
4555 		}
4556 	}
4557 	/* Done */
4558 	else {
4559 		err = -EINVAL;
4560 		if (unlikely(req->tp_frame_nr))
4561 			goto out;
4562 	}
4563 
4564 
4565 	/* Detach socket from network */
4566 	spin_lock(&po->bind_lock);
4567 	was_running = packet_sock_flag(po, PACKET_SOCK_RUNNING);
4568 	num = po->num;
4569 	if (was_running) {
4570 		WRITE_ONCE(po->num, 0);
4571 		__unregister_prot_hook(sk, false);
4572 	}
4573 	spin_unlock(&po->bind_lock);
4574 
4575 	synchronize_net();
4576 
4577 	err = -EBUSY;
4578 	mutex_lock(&po->pg_vec_lock);
4579 	if (closing || atomic_long_read(&po->mapped) == 0) {
4580 		err = 0;
4581 		spin_lock_bh(&rb_queue->lock);
4582 		swap(rb->pg_vec, pg_vec);
4583 		if (po->tp_version <= TPACKET_V2)
4584 			swap(rb->rx_owner_map, rx_owner_map);
4585 		rb->frame_max = (req->tp_frame_nr - 1);
4586 		rb->head = 0;
4587 		rb->frame_size = req->tp_frame_size;
4588 		spin_unlock_bh(&rb_queue->lock);
4589 
4590 		swap(rb->pg_vec_order, order);
4591 		swap(rb->pg_vec_len, req->tp_block_nr);
4592 
4593 		rb->pg_vec_pages = req->tp_block_size/PAGE_SIZE;
4594 		po->prot_hook.func = (po->rx_ring.pg_vec) ?
4595 						tpacket_rcv : packet_rcv;
4596 		skb_queue_purge(rb_queue);
4597 		if (atomic_long_read(&po->mapped))
4598 			pr_err("packet_mmap: vma is busy: %ld\n",
4599 			       atomic_long_read(&po->mapped));
4600 	}
4601 	mutex_unlock(&po->pg_vec_lock);
4602 
4603 	spin_lock(&po->bind_lock);
4604 	if (was_running) {
4605 		WRITE_ONCE(po->num, num);
4606 		register_prot_hook(sk);
4607 	}
4608 	spin_unlock(&po->bind_lock);
4609 	if (pg_vec && (po->tp_version > TPACKET_V2)) {
4610 		/* Because we don't support block-based V3 on tx-ring */
4611 		if (!tx_ring)
4612 			prb_shutdown_retire_blk_timer(po, rb_queue);
4613 	}
4614 
4615 out_free_pg_vec:
4616 	if (pg_vec) {
4617 		bitmap_free(rx_owner_map);
4618 		free_pg_vec(pg_vec, order, req->tp_block_nr);
4619 	}
4620 out:
4621 	return err;
4622 }
4623 
packet_mmap(struct file * file,struct socket * sock,struct vm_area_struct * vma)4624 static int packet_mmap(struct file *file, struct socket *sock,
4625 		struct vm_area_struct *vma)
4626 {
4627 	struct sock *sk = sock->sk;
4628 	struct packet_sock *po = pkt_sk(sk);
4629 	unsigned long size, expected_size;
4630 	struct packet_ring_buffer *rb;
4631 	unsigned long start;
4632 	int err = -EINVAL;
4633 	int i;
4634 
4635 	if (vma->vm_pgoff)
4636 		return -EINVAL;
4637 
4638 	mutex_lock(&po->pg_vec_lock);
4639 
4640 	expected_size = 0;
4641 	for (rb = &po->rx_ring; rb <= &po->tx_ring; rb++) {
4642 		if (rb->pg_vec) {
4643 			expected_size += rb->pg_vec_len
4644 						* rb->pg_vec_pages
4645 						* PAGE_SIZE;
4646 		}
4647 	}
4648 
4649 	if (expected_size == 0)
4650 		goto out;
4651 
4652 	size = vma->vm_end - vma->vm_start;
4653 	if (size != expected_size)
4654 		goto out;
4655 
4656 	start = vma->vm_start;
4657 	for (rb = &po->rx_ring; rb <= &po->tx_ring; rb++) {
4658 		if (rb->pg_vec == NULL)
4659 			continue;
4660 
4661 		for (i = 0; i < rb->pg_vec_len; i++) {
4662 			struct page *page;
4663 			void *kaddr = rb->pg_vec[i].buffer;
4664 			int pg_num;
4665 
4666 			for (pg_num = 0; pg_num < rb->pg_vec_pages; pg_num++) {
4667 				page = pgv_to_page(kaddr);
4668 				err = vm_insert_page(vma, start, page);
4669 				if (unlikely(err))
4670 					goto out;
4671 				start += PAGE_SIZE;
4672 				kaddr += PAGE_SIZE;
4673 			}
4674 		}
4675 	}
4676 
4677 	atomic_long_inc(&po->mapped);
4678 	vma->vm_ops = &packet_mmap_ops;
4679 	err = 0;
4680 
4681 out:
4682 	mutex_unlock(&po->pg_vec_lock);
4683 	return err;
4684 }
4685 
4686 static const struct proto_ops packet_ops_spkt = {
4687 	.family =	PF_PACKET,
4688 	.owner =	THIS_MODULE,
4689 	.release =	packet_release,
4690 	.bind =		packet_bind_spkt,
4691 	.connect =	sock_no_connect,
4692 	.socketpair =	sock_no_socketpair,
4693 	.accept =	sock_no_accept,
4694 	.getname =	packet_getname_spkt,
4695 	.poll =		datagram_poll,
4696 	.ioctl =	packet_ioctl,
4697 	.gettstamp =	sock_gettstamp,
4698 	.listen =	sock_no_listen,
4699 	.shutdown =	sock_no_shutdown,
4700 	.sendmsg =	packet_sendmsg_spkt,
4701 	.recvmsg =	packet_recvmsg,
4702 	.mmap =		sock_no_mmap,
4703 };
4704 
4705 static const struct proto_ops packet_ops = {
4706 	.family =	PF_PACKET,
4707 	.owner =	THIS_MODULE,
4708 	.release =	packet_release,
4709 	.bind =		packet_bind,
4710 	.connect =	sock_no_connect,
4711 	.socketpair =	sock_no_socketpair,
4712 	.accept =	sock_no_accept,
4713 	.getname =	packet_getname,
4714 	.poll =		packet_poll,
4715 	.ioctl =	packet_ioctl,
4716 	.gettstamp =	sock_gettstamp,
4717 	.listen =	sock_no_listen,
4718 	.shutdown =	sock_no_shutdown,
4719 	.setsockopt =	packet_setsockopt,
4720 	.getsockopt =	packet_getsockopt,
4721 	.sendmsg =	packet_sendmsg,
4722 	.recvmsg =	packet_recvmsg,
4723 	.mmap =		packet_mmap,
4724 };
4725 
4726 static const struct net_proto_family packet_family_ops = {
4727 	.family =	PF_PACKET,
4728 	.create =	packet_create,
4729 	.owner	=	THIS_MODULE,
4730 };
4731 
4732 static struct notifier_block packet_netdev_notifier = {
4733 	.notifier_call =	packet_notifier,
4734 };
4735 
4736 #ifdef CONFIG_PROC_FS
4737 
packet_seq_start(struct seq_file * seq,loff_t * pos)4738 static void *packet_seq_start(struct seq_file *seq, loff_t *pos)
4739 	__acquires(RCU)
4740 {
4741 	struct net *net = seq_file_net(seq);
4742 
4743 	rcu_read_lock();
4744 	return seq_hlist_start_head_rcu(&net->packet.sklist, *pos);
4745 }
4746 
packet_seq_next(struct seq_file * seq,void * v,loff_t * pos)4747 static void *packet_seq_next(struct seq_file *seq, void *v, loff_t *pos)
4748 {
4749 	struct net *net = seq_file_net(seq);
4750 	return seq_hlist_next_rcu(v, &net->packet.sklist, pos);
4751 }
4752 
packet_seq_stop(struct seq_file * seq,void * v)4753 static void packet_seq_stop(struct seq_file *seq, void *v)
4754 	__releases(RCU)
4755 {
4756 	rcu_read_unlock();
4757 }
4758 
packet_seq_show(struct seq_file * seq,void * v)4759 static int packet_seq_show(struct seq_file *seq, void *v)
4760 {
4761 	if (v == SEQ_START_TOKEN)
4762 		seq_printf(seq,
4763 			   "%*sRefCnt Type Proto  Iface R Rmem   User   Inode\n",
4764 			   IS_ENABLED(CONFIG_64BIT) ? -17 : -9, "sk");
4765 	else {
4766 		struct sock *s = sk_entry(v);
4767 		const struct packet_sock *po = pkt_sk(s);
4768 
4769 		seq_printf(seq,
4770 			   "%pK %-6d %-4d %04x   %-5d %1d %-6u %-6u %-6lu\n",
4771 			   s,
4772 			   refcount_read(&s->sk_refcnt),
4773 			   s->sk_type,
4774 			   ntohs(READ_ONCE(po->num)),
4775 			   READ_ONCE(po->ifindex),
4776 			   packet_sock_flag(po, PACKET_SOCK_RUNNING),
4777 			   atomic_read(&s->sk_rmem_alloc),
4778 			   from_kuid_munged(seq_user_ns(seq), sock_i_uid(s)),
4779 			   sock_i_ino(s));
4780 	}
4781 
4782 	return 0;
4783 }
4784 
4785 static const struct seq_operations packet_seq_ops = {
4786 	.start	= packet_seq_start,
4787 	.next	= packet_seq_next,
4788 	.stop	= packet_seq_stop,
4789 	.show	= packet_seq_show,
4790 };
4791 #endif
4792 
packet_net_init(struct net * net)4793 static int __net_init packet_net_init(struct net *net)
4794 {
4795 	mutex_init(&net->packet.sklist_lock);
4796 	INIT_HLIST_HEAD(&net->packet.sklist);
4797 
4798 #ifdef CONFIG_PROC_FS
4799 	if (!proc_create_net("packet", 0, net->proc_net, &packet_seq_ops,
4800 			sizeof(struct seq_net_private)))
4801 		return -ENOMEM;
4802 #endif /* CONFIG_PROC_FS */
4803 
4804 	return 0;
4805 }
4806 
packet_net_exit(struct net * net)4807 static void __net_exit packet_net_exit(struct net *net)
4808 {
4809 	remove_proc_entry("packet", net->proc_net);
4810 	WARN_ON_ONCE(!hlist_empty(&net->packet.sklist));
4811 }
4812 
4813 static struct pernet_operations packet_net_ops = {
4814 	.init = packet_net_init,
4815 	.exit = packet_net_exit,
4816 };
4817 
4818 
packet_exit(void)4819 static void __exit packet_exit(void)
4820 {
4821 	sock_unregister(PF_PACKET);
4822 	proto_unregister(&packet_proto);
4823 	unregister_netdevice_notifier(&packet_netdev_notifier);
4824 	unregister_pernet_subsys(&packet_net_ops);
4825 }
4826 
packet_init(void)4827 static int __init packet_init(void)
4828 {
4829 	int rc;
4830 
4831 	rc = register_pernet_subsys(&packet_net_ops);
4832 	if (rc)
4833 		goto out;
4834 	rc = register_netdevice_notifier(&packet_netdev_notifier);
4835 	if (rc)
4836 		goto out_pernet;
4837 	rc = proto_register(&packet_proto, 0);
4838 	if (rc)
4839 		goto out_notifier;
4840 	rc = sock_register(&packet_family_ops);
4841 	if (rc)
4842 		goto out_proto;
4843 
4844 	return 0;
4845 
4846 out_proto:
4847 	proto_unregister(&packet_proto);
4848 out_notifier:
4849 	unregister_netdevice_notifier(&packet_netdev_notifier);
4850 out_pernet:
4851 	unregister_pernet_subsys(&packet_net_ops);
4852 out:
4853 	return rc;
4854 }
4855 
4856 module_init(packet_init);
4857 module_exit(packet_exit);
4858 MODULE_LICENSE("GPL");
4859 MODULE_ALIAS_NETPROTO(PF_PACKET);
4860