xref: /openbmc/linux/net/core/sock.c (revision d198b34f3855eee2571dda03eea75a09c7c31480)
1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /*
3  * INET		An implementation of the TCP/IP protocol suite for the LINUX
4  *		operating system.  INET is implemented using the  BSD Socket
5  *		interface as the means of communication with the user level.
6  *
7  *		Generic socket support routines. Memory allocators, socket lock/release
8  *		handler for protocols to use and generic option handler.
9  *
10  * Authors:	Ross Biro
11  *		Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
12  *		Florian La Roche, <flla@stud.uni-sb.de>
13  *		Alan Cox, <A.Cox@swansea.ac.uk>
14  *
15  * Fixes:
16  *		Alan Cox	: 	Numerous verify_area() problems
17  *		Alan Cox	:	Connecting on a connecting socket
18  *					now returns an error for tcp.
19  *		Alan Cox	:	sock->protocol is set correctly.
20  *					and is not sometimes left as 0.
21  *		Alan Cox	:	connect handles icmp errors on a
22  *					connect properly. Unfortunately there
23  *					is a restart syscall nasty there. I
24  *					can't match BSD without hacking the C
25  *					library. Ideas urgently sought!
26  *		Alan Cox	:	Disallow bind() to addresses that are
27  *					not ours - especially broadcast ones!!
28  *		Alan Cox	:	Socket 1024 _IS_ ok for users. (fencepost)
29  *		Alan Cox	:	sock_wfree/sock_rfree don't destroy sockets,
30  *					instead they leave that for the DESTROY timer.
31  *		Alan Cox	:	Clean up error flag in accept
32  *		Alan Cox	:	TCP ack handling is buggy, the DESTROY timer
33  *					was buggy. Put a remove_sock() in the handler
34  *					for memory when we hit 0. Also altered the timer
35  *					code. The ACK stuff can wait and needs major
36  *					TCP layer surgery.
37  *		Alan Cox	:	Fixed TCP ack bug, removed remove sock
38  *					and fixed timer/inet_bh race.
39  *		Alan Cox	:	Added zapped flag for TCP
40  *		Alan Cox	:	Move kfree_skb into skbuff.c and tidied up surplus code
41  *		Alan Cox	:	for new sk_buff allocations wmalloc/rmalloc now call alloc_skb
42  *		Alan Cox	:	kfree_s calls now are kfree_skbmem so we can track skb resources
43  *		Alan Cox	:	Supports socket option broadcast now as does udp. Packet and raw need fixing.
44  *		Alan Cox	:	Added RCVBUF,SNDBUF size setting. It suddenly occurred to me how easy it was so...
45  *		Rick Sladkey	:	Relaxed UDP rules for matching packets.
46  *		C.E.Hawkins	:	IFF_PROMISC/SIOCGHWADDR support
47  *	Pauline Middelink	:	identd support
48  *		Alan Cox	:	Fixed connect() taking signals I think.
49  *		Alan Cox	:	SO_LINGER supported
50  *		Alan Cox	:	Error reporting fixes
51  *		Anonymous	:	inet_create tidied up (sk->reuse setting)
52  *		Alan Cox	:	inet sockets don't set sk->type!
53  *		Alan Cox	:	Split socket option code
54  *		Alan Cox	:	Callbacks
55  *		Alan Cox	:	Nagle flag for Charles & Johannes stuff
56  *		Alex		:	Removed restriction on inet fioctl
57  *		Alan Cox	:	Splitting INET from NET core
58  *		Alan Cox	:	Fixed bogus SO_TYPE handling in getsockopt()
59  *		Adam Caldwell	:	Missing return in SO_DONTROUTE/SO_DEBUG code
60  *		Alan Cox	:	Split IP from generic code
61  *		Alan Cox	:	New kfree_skbmem()
62  *		Alan Cox	:	Make SO_DEBUG superuser only.
63  *		Alan Cox	:	Allow anyone to clear SO_DEBUG
64  *					(compatibility fix)
65  *		Alan Cox	:	Added optimistic memory grabbing for AF_UNIX throughput.
66  *		Alan Cox	:	Allocator for a socket is settable.
67  *		Alan Cox	:	SO_ERROR includes soft errors.
68  *		Alan Cox	:	Allow NULL arguments on some SO_ opts
69  *		Alan Cox	: 	Generic socket allocation to make hooks
70  *					easier (suggested by Craig Metz).
71  *		Michael Pall	:	SO_ERROR returns positive errno again
72  *              Steve Whitehouse:       Added default destructor to free
73  *                                      protocol private data.
74  *              Steve Whitehouse:       Added various other default routines
75  *                                      common to several socket families.
76  *              Chris Evans     :       Call suser() check last on F_SETOWN
77  *		Jay Schulist	:	Added SO_ATTACH_FILTER and SO_DETACH_FILTER.
78  *		Andi Kleen	:	Add sock_kmalloc()/sock_kfree_s()
79  *		Andi Kleen	:	Fix write_space callback
80  *		Chris Evans	:	Security fixes - signedness again
81  *		Arnaldo C. Melo :       cleanups, use skb_queue_purge
82  *
83  * To Fix:
84  */
85 
86 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
87 
88 #include <asm/unaligned.h>
89 #include <linux/capability.h>
90 #include <linux/errno.h>
91 #include <linux/errqueue.h>
92 #include <linux/types.h>
93 #include <linux/socket.h>
94 #include <linux/in.h>
95 #include <linux/kernel.h>
96 #include <linux/module.h>
97 #include <linux/proc_fs.h>
98 #include <linux/seq_file.h>
99 #include <linux/sched.h>
100 #include <linux/sched/mm.h>
101 #include <linux/timer.h>
102 #include <linux/string.h>
103 #include <linux/sockios.h>
104 #include <linux/net.h>
105 #include <linux/mm.h>
106 #include <linux/slab.h>
107 #include <linux/interrupt.h>
108 #include <linux/poll.h>
109 #include <linux/tcp.h>
110 #include <linux/init.h>
111 #include <linux/highmem.h>
112 #include <linux/user_namespace.h>
113 #include <linux/static_key.h>
114 #include <linux/memcontrol.h>
115 #include <linux/prefetch.h>
116 
117 #include <linux/uaccess.h>
118 
119 #include <linux/netdevice.h>
120 #include <net/protocol.h>
121 #include <linux/skbuff.h>
122 #include <net/net_namespace.h>
123 #include <net/request_sock.h>
124 #include <net/sock.h>
125 #include <linux/net_tstamp.h>
126 #include <net/xfrm.h>
127 #include <linux/ipsec.h>
128 #include <net/cls_cgroup.h>
129 #include <net/netprio_cgroup.h>
130 #include <linux/sock_diag.h>
131 
132 #include <linux/filter.h>
133 #include <net/sock_reuseport.h>
134 #include <net/bpf_sk_storage.h>
135 
136 #include <trace/events/sock.h>
137 
138 #include <net/tcp.h>
139 #include <net/busy_poll.h>
140 
141 static DEFINE_MUTEX(proto_list_mutex);
142 static LIST_HEAD(proto_list);
143 
144 static void sock_inuse_add(struct net *net, int val);
145 
146 /**
147  * sk_ns_capable - General socket capability test
148  * @sk: Socket to use a capability on or through
149  * @user_ns: The user namespace of the capability to use
150  * @cap: The capability to use
151  *
152  * Test to see if the opener of the socket had when the socket was
153  * created and the current process has the capability @cap in the user
154  * namespace @user_ns.
155  */
156 bool sk_ns_capable(const struct sock *sk,
157 		   struct user_namespace *user_ns, int cap)
158 {
159 	return file_ns_capable(sk->sk_socket->file, user_ns, cap) &&
160 		ns_capable(user_ns, cap);
161 }
162 EXPORT_SYMBOL(sk_ns_capable);
163 
164 /**
165  * sk_capable - Socket global capability test
166  * @sk: Socket to use a capability on or through
167  * @cap: The global capability to use
168  *
169  * Test to see if the opener of the socket had when the socket was
170  * created and the current process has the capability @cap in all user
171  * namespaces.
172  */
173 bool sk_capable(const struct sock *sk, int cap)
174 {
175 	return sk_ns_capable(sk, &init_user_ns, cap);
176 }
177 EXPORT_SYMBOL(sk_capable);
178 
179 /**
180  * sk_net_capable - Network namespace socket capability test
181  * @sk: Socket to use a capability on or through
182  * @cap: The capability to use
183  *
184  * Test to see if the opener of the socket had when the socket was created
185  * and the current process has the capability @cap over the network namespace
186  * the socket is a member of.
187  */
188 bool sk_net_capable(const struct sock *sk, int cap)
189 {
190 	return sk_ns_capable(sk, sock_net(sk)->user_ns, cap);
191 }
192 EXPORT_SYMBOL(sk_net_capable);
193 
194 /*
195  * Each address family might have different locking rules, so we have
196  * one slock key per address family and separate keys for internal and
197  * userspace sockets.
198  */
199 static struct lock_class_key af_family_keys[AF_MAX];
200 static struct lock_class_key af_family_kern_keys[AF_MAX];
201 static struct lock_class_key af_family_slock_keys[AF_MAX];
202 static struct lock_class_key af_family_kern_slock_keys[AF_MAX];
203 
204 /*
205  * Make lock validator output more readable. (we pre-construct these
206  * strings build-time, so that runtime initialization of socket
207  * locks is fast):
208  */
209 
210 #define _sock_locks(x)						  \
211   x "AF_UNSPEC",	x "AF_UNIX"     ,	x "AF_INET"     , \
212   x "AF_AX25"  ,	x "AF_IPX"      ,	x "AF_APPLETALK", \
213   x "AF_NETROM",	x "AF_BRIDGE"   ,	x "AF_ATMPVC"   , \
214   x "AF_X25"   ,	x "AF_INET6"    ,	x "AF_ROSE"     , \
215   x "AF_DECnet",	x "AF_NETBEUI"  ,	x "AF_SECURITY" , \
216   x "AF_KEY"   ,	x "AF_NETLINK"  ,	x "AF_PACKET"   , \
217   x "AF_ASH"   ,	x "AF_ECONET"   ,	x "AF_ATMSVC"   , \
218   x "AF_RDS"   ,	x "AF_SNA"      ,	x "AF_IRDA"     , \
219   x "AF_PPPOX" ,	x "AF_WANPIPE"  ,	x "AF_LLC"      , \
220   x "27"       ,	x "28"          ,	x "AF_CAN"      , \
221   x "AF_TIPC"  ,	x "AF_BLUETOOTH",	x "IUCV"        , \
222   x "AF_RXRPC" ,	x "AF_ISDN"     ,	x "AF_PHONET"   , \
223   x "AF_IEEE802154",	x "AF_CAIF"	,	x "AF_ALG"      , \
224   x "AF_NFC"   ,	x "AF_VSOCK"    ,	x "AF_KCM"      , \
225   x "AF_QIPCRTR",	x "AF_SMC"	,	x "AF_XDP"	, \
226   x "AF_MAX"
227 
228 static const char *const af_family_key_strings[AF_MAX+1] = {
229 	_sock_locks("sk_lock-")
230 };
231 static const char *const af_family_slock_key_strings[AF_MAX+1] = {
232 	_sock_locks("slock-")
233 };
234 static const char *const af_family_clock_key_strings[AF_MAX+1] = {
235 	_sock_locks("clock-")
236 };
237 
238 static const char *const af_family_kern_key_strings[AF_MAX+1] = {
239 	_sock_locks("k-sk_lock-")
240 };
241 static const char *const af_family_kern_slock_key_strings[AF_MAX+1] = {
242 	_sock_locks("k-slock-")
243 };
244 static const char *const af_family_kern_clock_key_strings[AF_MAX+1] = {
245 	_sock_locks("k-clock-")
246 };
247 static const char *const af_family_rlock_key_strings[AF_MAX+1] = {
248 	_sock_locks("rlock-")
249 };
250 static const char *const af_family_wlock_key_strings[AF_MAX+1] = {
251 	_sock_locks("wlock-")
252 };
253 static const char *const af_family_elock_key_strings[AF_MAX+1] = {
254 	_sock_locks("elock-")
255 };
256 
257 /*
258  * sk_callback_lock and sk queues locking rules are per-address-family,
259  * so split the lock classes by using a per-AF key:
260  */
261 static struct lock_class_key af_callback_keys[AF_MAX];
262 static struct lock_class_key af_rlock_keys[AF_MAX];
263 static struct lock_class_key af_wlock_keys[AF_MAX];
264 static struct lock_class_key af_elock_keys[AF_MAX];
265 static struct lock_class_key af_kern_callback_keys[AF_MAX];
266 
267 /* Run time adjustable parameters. */
268 __u32 sysctl_wmem_max __read_mostly = SK_WMEM_MAX;
269 EXPORT_SYMBOL(sysctl_wmem_max);
270 __u32 sysctl_rmem_max __read_mostly = SK_RMEM_MAX;
271 EXPORT_SYMBOL(sysctl_rmem_max);
272 __u32 sysctl_wmem_default __read_mostly = SK_WMEM_MAX;
273 __u32 sysctl_rmem_default __read_mostly = SK_RMEM_MAX;
274 
275 /* Maximal space eaten by iovec or ancillary data plus some space */
276 int sysctl_optmem_max __read_mostly = sizeof(unsigned long)*(2*UIO_MAXIOV+512);
277 EXPORT_SYMBOL(sysctl_optmem_max);
278 
279 int sysctl_tstamp_allow_data __read_mostly = 1;
280 
281 DEFINE_STATIC_KEY_FALSE(memalloc_socks_key);
282 EXPORT_SYMBOL_GPL(memalloc_socks_key);
283 
284 /**
285  * sk_set_memalloc - sets %SOCK_MEMALLOC
286  * @sk: socket to set it on
287  *
288  * Set %SOCK_MEMALLOC on a socket for access to emergency reserves.
289  * It's the responsibility of the admin to adjust min_free_kbytes
290  * to meet the requirements
291  */
292 void sk_set_memalloc(struct sock *sk)
293 {
294 	sock_set_flag(sk, SOCK_MEMALLOC);
295 	sk->sk_allocation |= __GFP_MEMALLOC;
296 	static_branch_inc(&memalloc_socks_key);
297 }
298 EXPORT_SYMBOL_GPL(sk_set_memalloc);
299 
300 void sk_clear_memalloc(struct sock *sk)
301 {
302 	sock_reset_flag(sk, SOCK_MEMALLOC);
303 	sk->sk_allocation &= ~__GFP_MEMALLOC;
304 	static_branch_dec(&memalloc_socks_key);
305 
306 	/*
307 	 * SOCK_MEMALLOC is allowed to ignore rmem limits to ensure forward
308 	 * progress of swapping. SOCK_MEMALLOC may be cleared while
309 	 * it has rmem allocations due to the last swapfile being deactivated
310 	 * but there is a risk that the socket is unusable due to exceeding
311 	 * the rmem limits. Reclaim the reserves and obey rmem limits again.
312 	 */
313 	sk_mem_reclaim(sk);
314 }
315 EXPORT_SYMBOL_GPL(sk_clear_memalloc);
316 
317 int __sk_backlog_rcv(struct sock *sk, struct sk_buff *skb)
318 {
319 	int ret;
320 	unsigned int noreclaim_flag;
321 
322 	/* these should have been dropped before queueing */
323 	BUG_ON(!sock_flag(sk, SOCK_MEMALLOC));
324 
325 	noreclaim_flag = memalloc_noreclaim_save();
326 	ret = sk->sk_backlog_rcv(sk, skb);
327 	memalloc_noreclaim_restore(noreclaim_flag);
328 
329 	return ret;
330 }
331 EXPORT_SYMBOL(__sk_backlog_rcv);
332 
333 static int sock_get_timeout(long timeo, void *optval, bool old_timeval)
334 {
335 	struct __kernel_sock_timeval tv;
336 
337 	if (timeo == MAX_SCHEDULE_TIMEOUT) {
338 		tv.tv_sec = 0;
339 		tv.tv_usec = 0;
340 	} else {
341 		tv.tv_sec = timeo / HZ;
342 		tv.tv_usec = ((timeo % HZ) * USEC_PER_SEC) / HZ;
343 	}
344 
345 	if (old_timeval && in_compat_syscall() && !COMPAT_USE_64BIT_TIME) {
346 		struct old_timeval32 tv32 = { tv.tv_sec, tv.tv_usec };
347 		*(struct old_timeval32 *)optval = tv32;
348 		return sizeof(tv32);
349 	}
350 
351 	if (old_timeval) {
352 		struct __kernel_old_timeval old_tv;
353 		old_tv.tv_sec = tv.tv_sec;
354 		old_tv.tv_usec = tv.tv_usec;
355 		*(struct __kernel_old_timeval *)optval = old_tv;
356 		return sizeof(old_tv);
357 	}
358 
359 	*(struct __kernel_sock_timeval *)optval = tv;
360 	return sizeof(tv);
361 }
362 
363 static int sock_set_timeout(long *timeo_p, char __user *optval, int optlen, bool old_timeval)
364 {
365 	struct __kernel_sock_timeval tv;
366 
367 	if (old_timeval && in_compat_syscall() && !COMPAT_USE_64BIT_TIME) {
368 		struct old_timeval32 tv32;
369 
370 		if (optlen < sizeof(tv32))
371 			return -EINVAL;
372 
373 		if (copy_from_user(&tv32, optval, sizeof(tv32)))
374 			return -EFAULT;
375 		tv.tv_sec = tv32.tv_sec;
376 		tv.tv_usec = tv32.tv_usec;
377 	} else if (old_timeval) {
378 		struct __kernel_old_timeval old_tv;
379 
380 		if (optlen < sizeof(old_tv))
381 			return -EINVAL;
382 		if (copy_from_user(&old_tv, optval, sizeof(old_tv)))
383 			return -EFAULT;
384 		tv.tv_sec = old_tv.tv_sec;
385 		tv.tv_usec = old_tv.tv_usec;
386 	} else {
387 		if (optlen < sizeof(tv))
388 			return -EINVAL;
389 		if (copy_from_user(&tv, optval, sizeof(tv)))
390 			return -EFAULT;
391 	}
392 	if (tv.tv_usec < 0 || tv.tv_usec >= USEC_PER_SEC)
393 		return -EDOM;
394 
395 	if (tv.tv_sec < 0) {
396 		static int warned __read_mostly;
397 
398 		*timeo_p = 0;
399 		if (warned < 10 && net_ratelimit()) {
400 			warned++;
401 			pr_info("%s: `%s' (pid %d) tries to set negative timeout\n",
402 				__func__, current->comm, task_pid_nr(current));
403 		}
404 		return 0;
405 	}
406 	*timeo_p = MAX_SCHEDULE_TIMEOUT;
407 	if (tv.tv_sec == 0 && tv.tv_usec == 0)
408 		return 0;
409 	if (tv.tv_sec < (MAX_SCHEDULE_TIMEOUT / HZ - 1))
410 		*timeo_p = tv.tv_sec * HZ + DIV_ROUND_UP((unsigned long)tv.tv_usec, USEC_PER_SEC / HZ);
411 	return 0;
412 }
413 
414 static void sock_warn_obsolete_bsdism(const char *name)
415 {
416 	static int warned;
417 	static char warncomm[TASK_COMM_LEN];
418 	if (strcmp(warncomm, current->comm) && warned < 5) {
419 		strcpy(warncomm,  current->comm);
420 		pr_warn("process `%s' is using obsolete %s SO_BSDCOMPAT\n",
421 			warncomm, name);
422 		warned++;
423 	}
424 }
425 
426 static bool sock_needs_netstamp(const struct sock *sk)
427 {
428 	switch (sk->sk_family) {
429 	case AF_UNSPEC:
430 	case AF_UNIX:
431 		return false;
432 	default:
433 		return true;
434 	}
435 }
436 
437 static void sock_disable_timestamp(struct sock *sk, unsigned long flags)
438 {
439 	if (sk->sk_flags & flags) {
440 		sk->sk_flags &= ~flags;
441 		if (sock_needs_netstamp(sk) &&
442 		    !(sk->sk_flags & SK_FLAGS_TIMESTAMP))
443 			net_disable_timestamp();
444 	}
445 }
446 
447 
448 int __sock_queue_rcv_skb(struct sock *sk, struct sk_buff *skb)
449 {
450 	unsigned long flags;
451 	struct sk_buff_head *list = &sk->sk_receive_queue;
452 
453 	if (atomic_read(&sk->sk_rmem_alloc) >= sk->sk_rcvbuf) {
454 		atomic_inc(&sk->sk_drops);
455 		trace_sock_rcvqueue_full(sk, skb);
456 		return -ENOMEM;
457 	}
458 
459 	if (!sk_rmem_schedule(sk, skb, skb->truesize)) {
460 		atomic_inc(&sk->sk_drops);
461 		return -ENOBUFS;
462 	}
463 
464 	skb->dev = NULL;
465 	skb_set_owner_r(skb, sk);
466 
467 	/* we escape from rcu protected region, make sure we dont leak
468 	 * a norefcounted dst
469 	 */
470 	skb_dst_force(skb);
471 
472 	spin_lock_irqsave(&list->lock, flags);
473 	sock_skb_set_dropcount(sk, skb);
474 	__skb_queue_tail(list, skb);
475 	spin_unlock_irqrestore(&list->lock, flags);
476 
477 	if (!sock_flag(sk, SOCK_DEAD))
478 		sk->sk_data_ready(sk);
479 	return 0;
480 }
481 EXPORT_SYMBOL(__sock_queue_rcv_skb);
482 
483 int sock_queue_rcv_skb(struct sock *sk, struct sk_buff *skb)
484 {
485 	int err;
486 
487 	err = sk_filter(sk, skb);
488 	if (err)
489 		return err;
490 
491 	return __sock_queue_rcv_skb(sk, skb);
492 }
493 EXPORT_SYMBOL(sock_queue_rcv_skb);
494 
495 int __sk_receive_skb(struct sock *sk, struct sk_buff *skb,
496 		     const int nested, unsigned int trim_cap, bool refcounted)
497 {
498 	int rc = NET_RX_SUCCESS;
499 
500 	if (sk_filter_trim_cap(sk, skb, trim_cap))
501 		goto discard_and_relse;
502 
503 	skb->dev = NULL;
504 
505 	if (sk_rcvqueues_full(sk, sk->sk_rcvbuf)) {
506 		atomic_inc(&sk->sk_drops);
507 		goto discard_and_relse;
508 	}
509 	if (nested)
510 		bh_lock_sock_nested(sk);
511 	else
512 		bh_lock_sock(sk);
513 	if (!sock_owned_by_user(sk)) {
514 		/*
515 		 * trylock + unlock semantics:
516 		 */
517 		mutex_acquire(&sk->sk_lock.dep_map, 0, 1, _RET_IP_);
518 
519 		rc = sk_backlog_rcv(sk, skb);
520 
521 		mutex_release(&sk->sk_lock.dep_map, _RET_IP_);
522 	} else if (sk_add_backlog(sk, skb, READ_ONCE(sk->sk_rcvbuf))) {
523 		bh_unlock_sock(sk);
524 		atomic_inc(&sk->sk_drops);
525 		goto discard_and_relse;
526 	}
527 
528 	bh_unlock_sock(sk);
529 out:
530 	if (refcounted)
531 		sock_put(sk);
532 	return rc;
533 discard_and_relse:
534 	kfree_skb(skb);
535 	goto out;
536 }
537 EXPORT_SYMBOL(__sk_receive_skb);
538 
539 struct dst_entry *__sk_dst_check(struct sock *sk, u32 cookie)
540 {
541 	struct dst_entry *dst = __sk_dst_get(sk);
542 
543 	if (dst && dst->obsolete && dst->ops->check(dst, cookie) == NULL) {
544 		sk_tx_queue_clear(sk);
545 		sk->sk_dst_pending_confirm = 0;
546 		RCU_INIT_POINTER(sk->sk_dst_cache, NULL);
547 		dst_release(dst);
548 		return NULL;
549 	}
550 
551 	return dst;
552 }
553 EXPORT_SYMBOL(__sk_dst_check);
554 
555 struct dst_entry *sk_dst_check(struct sock *sk, u32 cookie)
556 {
557 	struct dst_entry *dst = sk_dst_get(sk);
558 
559 	if (dst && dst->obsolete && dst->ops->check(dst, cookie) == NULL) {
560 		sk_dst_reset(sk);
561 		dst_release(dst);
562 		return NULL;
563 	}
564 
565 	return dst;
566 }
567 EXPORT_SYMBOL(sk_dst_check);
568 
569 static int sock_setbindtodevice_locked(struct sock *sk, int ifindex)
570 {
571 	int ret = -ENOPROTOOPT;
572 #ifdef CONFIG_NETDEVICES
573 	struct net *net = sock_net(sk);
574 
575 	/* Sorry... */
576 	ret = -EPERM;
577 	if (!ns_capable(net->user_ns, CAP_NET_RAW))
578 		goto out;
579 
580 	ret = -EINVAL;
581 	if (ifindex < 0)
582 		goto out;
583 
584 	sk->sk_bound_dev_if = ifindex;
585 	if (sk->sk_prot->rehash)
586 		sk->sk_prot->rehash(sk);
587 	sk_dst_reset(sk);
588 
589 	ret = 0;
590 
591 out:
592 #endif
593 
594 	return ret;
595 }
596 
597 static int sock_setbindtodevice(struct sock *sk, char __user *optval,
598 				int optlen)
599 {
600 	int ret = -ENOPROTOOPT;
601 #ifdef CONFIG_NETDEVICES
602 	struct net *net = sock_net(sk);
603 	char devname[IFNAMSIZ];
604 	int index;
605 
606 	ret = -EINVAL;
607 	if (optlen < 0)
608 		goto out;
609 
610 	/* Bind this socket to a particular device like "eth0",
611 	 * as specified in the passed interface name. If the
612 	 * name is "" or the option length is zero the socket
613 	 * is not bound.
614 	 */
615 	if (optlen > IFNAMSIZ - 1)
616 		optlen = IFNAMSIZ - 1;
617 	memset(devname, 0, sizeof(devname));
618 
619 	ret = -EFAULT;
620 	if (copy_from_user(devname, optval, optlen))
621 		goto out;
622 
623 	index = 0;
624 	if (devname[0] != '\0') {
625 		struct net_device *dev;
626 
627 		rcu_read_lock();
628 		dev = dev_get_by_name_rcu(net, devname);
629 		if (dev)
630 			index = dev->ifindex;
631 		rcu_read_unlock();
632 		ret = -ENODEV;
633 		if (!dev)
634 			goto out;
635 	}
636 
637 	lock_sock(sk);
638 	ret = sock_setbindtodevice_locked(sk, index);
639 	release_sock(sk);
640 
641 out:
642 #endif
643 
644 	return ret;
645 }
646 
647 static int sock_getbindtodevice(struct sock *sk, char __user *optval,
648 				int __user *optlen, int len)
649 {
650 	int ret = -ENOPROTOOPT;
651 #ifdef CONFIG_NETDEVICES
652 	struct net *net = sock_net(sk);
653 	char devname[IFNAMSIZ];
654 
655 	if (sk->sk_bound_dev_if == 0) {
656 		len = 0;
657 		goto zero;
658 	}
659 
660 	ret = -EINVAL;
661 	if (len < IFNAMSIZ)
662 		goto out;
663 
664 	ret = netdev_get_name(net, devname, sk->sk_bound_dev_if);
665 	if (ret)
666 		goto out;
667 
668 	len = strlen(devname) + 1;
669 
670 	ret = -EFAULT;
671 	if (copy_to_user(optval, devname, len))
672 		goto out;
673 
674 zero:
675 	ret = -EFAULT;
676 	if (put_user(len, optlen))
677 		goto out;
678 
679 	ret = 0;
680 
681 out:
682 #endif
683 
684 	return ret;
685 }
686 
687 static inline void sock_valbool_flag(struct sock *sk, enum sock_flags bit,
688 				     int valbool)
689 {
690 	if (valbool)
691 		sock_set_flag(sk, bit);
692 	else
693 		sock_reset_flag(sk, bit);
694 }
695 
696 bool sk_mc_loop(struct sock *sk)
697 {
698 	if (dev_recursion_level())
699 		return false;
700 	if (!sk)
701 		return true;
702 	switch (sk->sk_family) {
703 	case AF_INET:
704 		return inet_sk(sk)->mc_loop;
705 #if IS_ENABLED(CONFIG_IPV6)
706 	case AF_INET6:
707 		return inet6_sk(sk)->mc_loop;
708 #endif
709 	}
710 	WARN_ON(1);
711 	return true;
712 }
713 EXPORT_SYMBOL(sk_mc_loop);
714 
715 /*
716  *	This is meant for all protocols to use and covers goings on
717  *	at the socket level. Everything here is generic.
718  */
719 
720 int sock_setsockopt(struct socket *sock, int level, int optname,
721 		    char __user *optval, unsigned int optlen)
722 {
723 	struct sock_txtime sk_txtime;
724 	struct sock *sk = sock->sk;
725 	int val;
726 	int valbool;
727 	struct linger ling;
728 	int ret = 0;
729 
730 	/*
731 	 *	Options without arguments
732 	 */
733 
734 	if (optname == SO_BINDTODEVICE)
735 		return sock_setbindtodevice(sk, optval, optlen);
736 
737 	if (optlen < sizeof(int))
738 		return -EINVAL;
739 
740 	if (get_user(val, (int __user *)optval))
741 		return -EFAULT;
742 
743 	valbool = val ? 1 : 0;
744 
745 	lock_sock(sk);
746 
747 	switch (optname) {
748 	case SO_DEBUG:
749 		if (val && !capable(CAP_NET_ADMIN))
750 			ret = -EACCES;
751 		else
752 			sock_valbool_flag(sk, SOCK_DBG, valbool);
753 		break;
754 	case SO_REUSEADDR:
755 		sk->sk_reuse = (valbool ? SK_CAN_REUSE : SK_NO_REUSE);
756 		break;
757 	case SO_REUSEPORT:
758 		sk->sk_reuseport = valbool;
759 		break;
760 	case SO_TYPE:
761 	case SO_PROTOCOL:
762 	case SO_DOMAIN:
763 	case SO_ERROR:
764 		ret = -ENOPROTOOPT;
765 		break;
766 	case SO_DONTROUTE:
767 		sock_valbool_flag(sk, SOCK_LOCALROUTE, valbool);
768 		sk_dst_reset(sk);
769 		break;
770 	case SO_BROADCAST:
771 		sock_valbool_flag(sk, SOCK_BROADCAST, valbool);
772 		break;
773 	case SO_SNDBUF:
774 		/* Don't error on this BSD doesn't and if you think
775 		 * about it this is right. Otherwise apps have to
776 		 * play 'guess the biggest size' games. RCVBUF/SNDBUF
777 		 * are treated in BSD as hints
778 		 */
779 		val = min_t(u32, val, sysctl_wmem_max);
780 set_sndbuf:
781 		/* Ensure val * 2 fits into an int, to prevent max_t()
782 		 * from treating it as a negative value.
783 		 */
784 		val = min_t(int, val, INT_MAX / 2);
785 		sk->sk_userlocks |= SOCK_SNDBUF_LOCK;
786 		WRITE_ONCE(sk->sk_sndbuf,
787 			   max_t(int, val * 2, SOCK_MIN_SNDBUF));
788 		/* Wake up sending tasks if we upped the value. */
789 		sk->sk_write_space(sk);
790 		break;
791 
792 	case SO_SNDBUFFORCE:
793 		if (!capable(CAP_NET_ADMIN)) {
794 			ret = -EPERM;
795 			break;
796 		}
797 
798 		/* No negative values (to prevent underflow, as val will be
799 		 * multiplied by 2).
800 		 */
801 		if (val < 0)
802 			val = 0;
803 		goto set_sndbuf;
804 
805 	case SO_RCVBUF:
806 		/* Don't error on this BSD doesn't and if you think
807 		 * about it this is right. Otherwise apps have to
808 		 * play 'guess the biggest size' games. RCVBUF/SNDBUF
809 		 * are treated in BSD as hints
810 		 */
811 		val = min_t(u32, val, sysctl_rmem_max);
812 set_rcvbuf:
813 		/* Ensure val * 2 fits into an int, to prevent max_t()
814 		 * from treating it as a negative value.
815 		 */
816 		val = min_t(int, val, INT_MAX / 2);
817 		sk->sk_userlocks |= SOCK_RCVBUF_LOCK;
818 		/*
819 		 * We double it on the way in to account for
820 		 * "struct sk_buff" etc. overhead.   Applications
821 		 * assume that the SO_RCVBUF setting they make will
822 		 * allow that much actual data to be received on that
823 		 * socket.
824 		 *
825 		 * Applications are unaware that "struct sk_buff" and
826 		 * other overheads allocate from the receive buffer
827 		 * during socket buffer allocation.
828 		 *
829 		 * And after considering the possible alternatives,
830 		 * returning the value we actually used in getsockopt
831 		 * is the most desirable behavior.
832 		 */
833 		WRITE_ONCE(sk->sk_rcvbuf,
834 			   max_t(int, val * 2, SOCK_MIN_RCVBUF));
835 		break;
836 
837 	case SO_RCVBUFFORCE:
838 		if (!capable(CAP_NET_ADMIN)) {
839 			ret = -EPERM;
840 			break;
841 		}
842 
843 		/* No negative values (to prevent underflow, as val will be
844 		 * multiplied by 2).
845 		 */
846 		if (val < 0)
847 			val = 0;
848 		goto set_rcvbuf;
849 
850 	case SO_KEEPALIVE:
851 		if (sk->sk_prot->keepalive)
852 			sk->sk_prot->keepalive(sk, valbool);
853 		sock_valbool_flag(sk, SOCK_KEEPOPEN, valbool);
854 		break;
855 
856 	case SO_OOBINLINE:
857 		sock_valbool_flag(sk, SOCK_URGINLINE, valbool);
858 		break;
859 
860 	case SO_NO_CHECK:
861 		sk->sk_no_check_tx = valbool;
862 		break;
863 
864 	case SO_PRIORITY:
865 		if ((val >= 0 && val <= 6) ||
866 		    ns_capable(sock_net(sk)->user_ns, CAP_NET_ADMIN))
867 			sk->sk_priority = val;
868 		else
869 			ret = -EPERM;
870 		break;
871 
872 	case SO_LINGER:
873 		if (optlen < sizeof(ling)) {
874 			ret = -EINVAL;	/* 1003.1g */
875 			break;
876 		}
877 		if (copy_from_user(&ling, optval, sizeof(ling))) {
878 			ret = -EFAULT;
879 			break;
880 		}
881 		if (!ling.l_onoff)
882 			sock_reset_flag(sk, SOCK_LINGER);
883 		else {
884 #if (BITS_PER_LONG == 32)
885 			if ((unsigned int)ling.l_linger >= MAX_SCHEDULE_TIMEOUT/HZ)
886 				sk->sk_lingertime = MAX_SCHEDULE_TIMEOUT;
887 			else
888 #endif
889 				sk->sk_lingertime = (unsigned int)ling.l_linger * HZ;
890 			sock_set_flag(sk, SOCK_LINGER);
891 		}
892 		break;
893 
894 	case SO_BSDCOMPAT:
895 		sock_warn_obsolete_bsdism("setsockopt");
896 		break;
897 
898 	case SO_PASSCRED:
899 		if (valbool)
900 			set_bit(SOCK_PASSCRED, &sock->flags);
901 		else
902 			clear_bit(SOCK_PASSCRED, &sock->flags);
903 		break;
904 
905 	case SO_TIMESTAMP_OLD:
906 	case SO_TIMESTAMP_NEW:
907 	case SO_TIMESTAMPNS_OLD:
908 	case SO_TIMESTAMPNS_NEW:
909 		if (valbool)  {
910 			if (optname == SO_TIMESTAMP_NEW || optname == SO_TIMESTAMPNS_NEW)
911 				sock_set_flag(sk, SOCK_TSTAMP_NEW);
912 			else
913 				sock_reset_flag(sk, SOCK_TSTAMP_NEW);
914 
915 			if (optname == SO_TIMESTAMP_OLD || optname == SO_TIMESTAMP_NEW)
916 				sock_reset_flag(sk, SOCK_RCVTSTAMPNS);
917 			else
918 				sock_set_flag(sk, SOCK_RCVTSTAMPNS);
919 			sock_set_flag(sk, SOCK_RCVTSTAMP);
920 			sock_enable_timestamp(sk, SOCK_TIMESTAMP);
921 		} else {
922 			sock_reset_flag(sk, SOCK_RCVTSTAMP);
923 			sock_reset_flag(sk, SOCK_RCVTSTAMPNS);
924 			sock_reset_flag(sk, SOCK_TSTAMP_NEW);
925 		}
926 		break;
927 
928 	case SO_TIMESTAMPING_NEW:
929 		sock_set_flag(sk, SOCK_TSTAMP_NEW);
930 		/* fall through */
931 	case SO_TIMESTAMPING_OLD:
932 		if (val & ~SOF_TIMESTAMPING_MASK) {
933 			ret = -EINVAL;
934 			break;
935 		}
936 
937 		if (val & SOF_TIMESTAMPING_OPT_ID &&
938 		    !(sk->sk_tsflags & SOF_TIMESTAMPING_OPT_ID)) {
939 			if (sk->sk_protocol == IPPROTO_TCP &&
940 			    sk->sk_type == SOCK_STREAM) {
941 				if ((1 << sk->sk_state) &
942 				    (TCPF_CLOSE | TCPF_LISTEN)) {
943 					ret = -EINVAL;
944 					break;
945 				}
946 				sk->sk_tskey = tcp_sk(sk)->snd_una;
947 			} else {
948 				sk->sk_tskey = 0;
949 			}
950 		}
951 
952 		if (val & SOF_TIMESTAMPING_OPT_STATS &&
953 		    !(val & SOF_TIMESTAMPING_OPT_TSONLY)) {
954 			ret = -EINVAL;
955 			break;
956 		}
957 
958 		sk->sk_tsflags = val;
959 		if (val & SOF_TIMESTAMPING_RX_SOFTWARE)
960 			sock_enable_timestamp(sk,
961 					      SOCK_TIMESTAMPING_RX_SOFTWARE);
962 		else {
963 			if (optname == SO_TIMESTAMPING_NEW)
964 				sock_reset_flag(sk, SOCK_TSTAMP_NEW);
965 
966 			sock_disable_timestamp(sk,
967 					       (1UL << SOCK_TIMESTAMPING_RX_SOFTWARE));
968 		}
969 		break;
970 
971 	case SO_RCVLOWAT:
972 		if (val < 0)
973 			val = INT_MAX;
974 		if (sock->ops->set_rcvlowat)
975 			ret = sock->ops->set_rcvlowat(sk, val);
976 		else
977 			WRITE_ONCE(sk->sk_rcvlowat, val ? : 1);
978 		break;
979 
980 	case SO_RCVTIMEO_OLD:
981 	case SO_RCVTIMEO_NEW:
982 		ret = sock_set_timeout(&sk->sk_rcvtimeo, optval, optlen, optname == SO_RCVTIMEO_OLD);
983 		break;
984 
985 	case SO_SNDTIMEO_OLD:
986 	case SO_SNDTIMEO_NEW:
987 		ret = sock_set_timeout(&sk->sk_sndtimeo, optval, optlen, optname == SO_SNDTIMEO_OLD);
988 		break;
989 
990 	case SO_ATTACH_FILTER:
991 		ret = -EINVAL;
992 		if (optlen == sizeof(struct sock_fprog)) {
993 			struct sock_fprog fprog;
994 
995 			ret = -EFAULT;
996 			if (copy_from_user(&fprog, optval, sizeof(fprog)))
997 				break;
998 
999 			ret = sk_attach_filter(&fprog, sk);
1000 		}
1001 		break;
1002 
1003 	case SO_ATTACH_BPF:
1004 		ret = -EINVAL;
1005 		if (optlen == sizeof(u32)) {
1006 			u32 ufd;
1007 
1008 			ret = -EFAULT;
1009 			if (copy_from_user(&ufd, optval, sizeof(ufd)))
1010 				break;
1011 
1012 			ret = sk_attach_bpf(ufd, sk);
1013 		}
1014 		break;
1015 
1016 	case SO_ATTACH_REUSEPORT_CBPF:
1017 		ret = -EINVAL;
1018 		if (optlen == sizeof(struct sock_fprog)) {
1019 			struct sock_fprog fprog;
1020 
1021 			ret = -EFAULT;
1022 			if (copy_from_user(&fprog, optval, sizeof(fprog)))
1023 				break;
1024 
1025 			ret = sk_reuseport_attach_filter(&fprog, sk);
1026 		}
1027 		break;
1028 
1029 	case SO_ATTACH_REUSEPORT_EBPF:
1030 		ret = -EINVAL;
1031 		if (optlen == sizeof(u32)) {
1032 			u32 ufd;
1033 
1034 			ret = -EFAULT;
1035 			if (copy_from_user(&ufd, optval, sizeof(ufd)))
1036 				break;
1037 
1038 			ret = sk_reuseport_attach_bpf(ufd, sk);
1039 		}
1040 		break;
1041 
1042 	case SO_DETACH_REUSEPORT_BPF:
1043 		ret = reuseport_detach_prog(sk);
1044 		break;
1045 
1046 	case SO_DETACH_FILTER:
1047 		ret = sk_detach_filter(sk);
1048 		break;
1049 
1050 	case SO_LOCK_FILTER:
1051 		if (sock_flag(sk, SOCK_FILTER_LOCKED) && !valbool)
1052 			ret = -EPERM;
1053 		else
1054 			sock_valbool_flag(sk, SOCK_FILTER_LOCKED, valbool);
1055 		break;
1056 
1057 	case SO_PASSSEC:
1058 		if (valbool)
1059 			set_bit(SOCK_PASSSEC, &sock->flags);
1060 		else
1061 			clear_bit(SOCK_PASSSEC, &sock->flags);
1062 		break;
1063 	case SO_MARK:
1064 		if (!ns_capable(sock_net(sk)->user_ns, CAP_NET_ADMIN)) {
1065 			ret = -EPERM;
1066 		} else if (val != sk->sk_mark) {
1067 			sk->sk_mark = val;
1068 			sk_dst_reset(sk);
1069 		}
1070 		break;
1071 
1072 	case SO_RXQ_OVFL:
1073 		sock_valbool_flag(sk, SOCK_RXQ_OVFL, valbool);
1074 		break;
1075 
1076 	case SO_WIFI_STATUS:
1077 		sock_valbool_flag(sk, SOCK_WIFI_STATUS, valbool);
1078 		break;
1079 
1080 	case SO_PEEK_OFF:
1081 		if (sock->ops->set_peek_off)
1082 			ret = sock->ops->set_peek_off(sk, val);
1083 		else
1084 			ret = -EOPNOTSUPP;
1085 		break;
1086 
1087 	case SO_NOFCS:
1088 		sock_valbool_flag(sk, SOCK_NOFCS, valbool);
1089 		break;
1090 
1091 	case SO_SELECT_ERR_QUEUE:
1092 		sock_valbool_flag(sk, SOCK_SELECT_ERR_QUEUE, valbool);
1093 		break;
1094 
1095 #ifdef CONFIG_NET_RX_BUSY_POLL
1096 	case SO_BUSY_POLL:
1097 		/* allow unprivileged users to decrease the value */
1098 		if ((val > sk->sk_ll_usec) && !capable(CAP_NET_ADMIN))
1099 			ret = -EPERM;
1100 		else {
1101 			if (val < 0)
1102 				ret = -EINVAL;
1103 			else
1104 				sk->sk_ll_usec = val;
1105 		}
1106 		break;
1107 #endif
1108 
1109 	case SO_MAX_PACING_RATE:
1110 		{
1111 		unsigned long ulval = (val == ~0U) ? ~0UL : val;
1112 
1113 		if (sizeof(ulval) != sizeof(val) &&
1114 		    optlen >= sizeof(ulval) &&
1115 		    get_user(ulval, (unsigned long __user *)optval)) {
1116 			ret = -EFAULT;
1117 			break;
1118 		}
1119 		if (ulval != ~0UL)
1120 			cmpxchg(&sk->sk_pacing_status,
1121 				SK_PACING_NONE,
1122 				SK_PACING_NEEDED);
1123 		sk->sk_max_pacing_rate = ulval;
1124 		sk->sk_pacing_rate = min(sk->sk_pacing_rate, ulval);
1125 		break;
1126 		}
1127 	case SO_INCOMING_CPU:
1128 		WRITE_ONCE(sk->sk_incoming_cpu, val);
1129 		break;
1130 
1131 	case SO_CNX_ADVICE:
1132 		if (val == 1)
1133 			dst_negative_advice(sk);
1134 		break;
1135 
1136 	case SO_ZEROCOPY:
1137 		if (sk->sk_family == PF_INET || sk->sk_family == PF_INET6) {
1138 			if (!((sk->sk_type == SOCK_STREAM &&
1139 			       sk->sk_protocol == IPPROTO_TCP) ||
1140 			      (sk->sk_type == SOCK_DGRAM &&
1141 			       sk->sk_protocol == IPPROTO_UDP)))
1142 				ret = -ENOTSUPP;
1143 		} else if (sk->sk_family != PF_RDS) {
1144 			ret = -ENOTSUPP;
1145 		}
1146 		if (!ret) {
1147 			if (val < 0 || val > 1)
1148 				ret = -EINVAL;
1149 			else
1150 				sock_valbool_flag(sk, SOCK_ZEROCOPY, valbool);
1151 		}
1152 		break;
1153 
1154 	case SO_TXTIME:
1155 		if (!ns_capable(sock_net(sk)->user_ns, CAP_NET_ADMIN)) {
1156 			ret = -EPERM;
1157 		} else if (optlen != sizeof(struct sock_txtime)) {
1158 			ret = -EINVAL;
1159 		} else if (copy_from_user(&sk_txtime, optval,
1160 			   sizeof(struct sock_txtime))) {
1161 			ret = -EFAULT;
1162 		} else if (sk_txtime.flags & ~SOF_TXTIME_FLAGS_MASK) {
1163 			ret = -EINVAL;
1164 		} else {
1165 			sock_valbool_flag(sk, SOCK_TXTIME, true);
1166 			sk->sk_clockid = sk_txtime.clockid;
1167 			sk->sk_txtime_deadline_mode =
1168 				!!(sk_txtime.flags & SOF_TXTIME_DEADLINE_MODE);
1169 			sk->sk_txtime_report_errors =
1170 				!!(sk_txtime.flags & SOF_TXTIME_REPORT_ERRORS);
1171 		}
1172 		break;
1173 
1174 	case SO_BINDTOIFINDEX:
1175 		ret = sock_setbindtodevice_locked(sk, val);
1176 		break;
1177 
1178 	default:
1179 		ret = -ENOPROTOOPT;
1180 		break;
1181 	}
1182 	release_sock(sk);
1183 	return ret;
1184 }
1185 EXPORT_SYMBOL(sock_setsockopt);
1186 
1187 
1188 static void cred_to_ucred(struct pid *pid, const struct cred *cred,
1189 			  struct ucred *ucred)
1190 {
1191 	ucred->pid = pid_vnr(pid);
1192 	ucred->uid = ucred->gid = -1;
1193 	if (cred) {
1194 		struct user_namespace *current_ns = current_user_ns();
1195 
1196 		ucred->uid = from_kuid_munged(current_ns, cred->euid);
1197 		ucred->gid = from_kgid_munged(current_ns, cred->egid);
1198 	}
1199 }
1200 
1201 static int groups_to_user(gid_t __user *dst, const struct group_info *src)
1202 {
1203 	struct user_namespace *user_ns = current_user_ns();
1204 	int i;
1205 
1206 	for (i = 0; i < src->ngroups; i++)
1207 		if (put_user(from_kgid_munged(user_ns, src->gid[i]), dst + i))
1208 			return -EFAULT;
1209 
1210 	return 0;
1211 }
1212 
1213 int sock_getsockopt(struct socket *sock, int level, int optname,
1214 		    char __user *optval, int __user *optlen)
1215 {
1216 	struct sock *sk = sock->sk;
1217 
1218 	union {
1219 		int val;
1220 		u64 val64;
1221 		unsigned long ulval;
1222 		struct linger ling;
1223 		struct old_timeval32 tm32;
1224 		struct __kernel_old_timeval tm;
1225 		struct  __kernel_sock_timeval stm;
1226 		struct sock_txtime txtime;
1227 	} v;
1228 
1229 	int lv = sizeof(int);
1230 	int len;
1231 
1232 	if (get_user(len, optlen))
1233 		return -EFAULT;
1234 	if (len < 0)
1235 		return -EINVAL;
1236 
1237 	memset(&v, 0, sizeof(v));
1238 
1239 	switch (optname) {
1240 	case SO_DEBUG:
1241 		v.val = sock_flag(sk, SOCK_DBG);
1242 		break;
1243 
1244 	case SO_DONTROUTE:
1245 		v.val = sock_flag(sk, SOCK_LOCALROUTE);
1246 		break;
1247 
1248 	case SO_BROADCAST:
1249 		v.val = sock_flag(sk, SOCK_BROADCAST);
1250 		break;
1251 
1252 	case SO_SNDBUF:
1253 		v.val = sk->sk_sndbuf;
1254 		break;
1255 
1256 	case SO_RCVBUF:
1257 		v.val = sk->sk_rcvbuf;
1258 		break;
1259 
1260 	case SO_REUSEADDR:
1261 		v.val = sk->sk_reuse;
1262 		break;
1263 
1264 	case SO_REUSEPORT:
1265 		v.val = sk->sk_reuseport;
1266 		break;
1267 
1268 	case SO_KEEPALIVE:
1269 		v.val = sock_flag(sk, SOCK_KEEPOPEN);
1270 		break;
1271 
1272 	case SO_TYPE:
1273 		v.val = sk->sk_type;
1274 		break;
1275 
1276 	case SO_PROTOCOL:
1277 		v.val = sk->sk_protocol;
1278 		break;
1279 
1280 	case SO_DOMAIN:
1281 		v.val = sk->sk_family;
1282 		break;
1283 
1284 	case SO_ERROR:
1285 		v.val = -sock_error(sk);
1286 		if (v.val == 0)
1287 			v.val = xchg(&sk->sk_err_soft, 0);
1288 		break;
1289 
1290 	case SO_OOBINLINE:
1291 		v.val = sock_flag(sk, SOCK_URGINLINE);
1292 		break;
1293 
1294 	case SO_NO_CHECK:
1295 		v.val = sk->sk_no_check_tx;
1296 		break;
1297 
1298 	case SO_PRIORITY:
1299 		v.val = sk->sk_priority;
1300 		break;
1301 
1302 	case SO_LINGER:
1303 		lv		= sizeof(v.ling);
1304 		v.ling.l_onoff	= sock_flag(sk, SOCK_LINGER);
1305 		v.ling.l_linger	= sk->sk_lingertime / HZ;
1306 		break;
1307 
1308 	case SO_BSDCOMPAT:
1309 		sock_warn_obsolete_bsdism("getsockopt");
1310 		break;
1311 
1312 	case SO_TIMESTAMP_OLD:
1313 		v.val = sock_flag(sk, SOCK_RCVTSTAMP) &&
1314 				!sock_flag(sk, SOCK_TSTAMP_NEW) &&
1315 				!sock_flag(sk, SOCK_RCVTSTAMPNS);
1316 		break;
1317 
1318 	case SO_TIMESTAMPNS_OLD:
1319 		v.val = sock_flag(sk, SOCK_RCVTSTAMPNS) && !sock_flag(sk, SOCK_TSTAMP_NEW);
1320 		break;
1321 
1322 	case SO_TIMESTAMP_NEW:
1323 		v.val = sock_flag(sk, SOCK_RCVTSTAMP) && sock_flag(sk, SOCK_TSTAMP_NEW);
1324 		break;
1325 
1326 	case SO_TIMESTAMPNS_NEW:
1327 		v.val = sock_flag(sk, SOCK_RCVTSTAMPNS) && sock_flag(sk, SOCK_TSTAMP_NEW);
1328 		break;
1329 
1330 	case SO_TIMESTAMPING_OLD:
1331 		v.val = sk->sk_tsflags;
1332 		break;
1333 
1334 	case SO_RCVTIMEO_OLD:
1335 	case SO_RCVTIMEO_NEW:
1336 		lv = sock_get_timeout(sk->sk_rcvtimeo, &v, SO_RCVTIMEO_OLD == optname);
1337 		break;
1338 
1339 	case SO_SNDTIMEO_OLD:
1340 	case SO_SNDTIMEO_NEW:
1341 		lv = sock_get_timeout(sk->sk_sndtimeo, &v, SO_SNDTIMEO_OLD == optname);
1342 		break;
1343 
1344 	case SO_RCVLOWAT:
1345 		v.val = sk->sk_rcvlowat;
1346 		break;
1347 
1348 	case SO_SNDLOWAT:
1349 		v.val = 1;
1350 		break;
1351 
1352 	case SO_PASSCRED:
1353 		v.val = !!test_bit(SOCK_PASSCRED, &sock->flags);
1354 		break;
1355 
1356 	case SO_PEERCRED:
1357 	{
1358 		struct ucred peercred;
1359 		if (len > sizeof(peercred))
1360 			len = sizeof(peercred);
1361 		cred_to_ucred(sk->sk_peer_pid, sk->sk_peer_cred, &peercred);
1362 		if (copy_to_user(optval, &peercred, len))
1363 			return -EFAULT;
1364 		goto lenout;
1365 	}
1366 
1367 	case SO_PEERGROUPS:
1368 	{
1369 		int ret, n;
1370 
1371 		if (!sk->sk_peer_cred)
1372 			return -ENODATA;
1373 
1374 		n = sk->sk_peer_cred->group_info->ngroups;
1375 		if (len < n * sizeof(gid_t)) {
1376 			len = n * sizeof(gid_t);
1377 			return put_user(len, optlen) ? -EFAULT : -ERANGE;
1378 		}
1379 		len = n * sizeof(gid_t);
1380 
1381 		ret = groups_to_user((gid_t __user *)optval,
1382 				     sk->sk_peer_cred->group_info);
1383 		if (ret)
1384 			return ret;
1385 		goto lenout;
1386 	}
1387 
1388 	case SO_PEERNAME:
1389 	{
1390 		char address[128];
1391 
1392 		lv = sock->ops->getname(sock, (struct sockaddr *)address, 2);
1393 		if (lv < 0)
1394 			return -ENOTCONN;
1395 		if (lv < len)
1396 			return -EINVAL;
1397 		if (copy_to_user(optval, address, len))
1398 			return -EFAULT;
1399 		goto lenout;
1400 	}
1401 
1402 	/* Dubious BSD thing... Probably nobody even uses it, but
1403 	 * the UNIX standard wants it for whatever reason... -DaveM
1404 	 */
1405 	case SO_ACCEPTCONN:
1406 		v.val = sk->sk_state == TCP_LISTEN;
1407 		break;
1408 
1409 	case SO_PASSSEC:
1410 		v.val = !!test_bit(SOCK_PASSSEC, &sock->flags);
1411 		break;
1412 
1413 	case SO_PEERSEC:
1414 		return security_socket_getpeersec_stream(sock, optval, optlen, len);
1415 
1416 	case SO_MARK:
1417 		v.val = sk->sk_mark;
1418 		break;
1419 
1420 	case SO_RXQ_OVFL:
1421 		v.val = sock_flag(sk, SOCK_RXQ_OVFL);
1422 		break;
1423 
1424 	case SO_WIFI_STATUS:
1425 		v.val = sock_flag(sk, SOCK_WIFI_STATUS);
1426 		break;
1427 
1428 	case SO_PEEK_OFF:
1429 		if (!sock->ops->set_peek_off)
1430 			return -EOPNOTSUPP;
1431 
1432 		v.val = sk->sk_peek_off;
1433 		break;
1434 	case SO_NOFCS:
1435 		v.val = sock_flag(sk, SOCK_NOFCS);
1436 		break;
1437 
1438 	case SO_BINDTODEVICE:
1439 		return sock_getbindtodevice(sk, optval, optlen, len);
1440 
1441 	case SO_GET_FILTER:
1442 		len = sk_get_filter(sk, (struct sock_filter __user *)optval, len);
1443 		if (len < 0)
1444 			return len;
1445 
1446 		goto lenout;
1447 
1448 	case SO_LOCK_FILTER:
1449 		v.val = sock_flag(sk, SOCK_FILTER_LOCKED);
1450 		break;
1451 
1452 	case SO_BPF_EXTENSIONS:
1453 		v.val = bpf_tell_extensions();
1454 		break;
1455 
1456 	case SO_SELECT_ERR_QUEUE:
1457 		v.val = sock_flag(sk, SOCK_SELECT_ERR_QUEUE);
1458 		break;
1459 
1460 #ifdef CONFIG_NET_RX_BUSY_POLL
1461 	case SO_BUSY_POLL:
1462 		v.val = sk->sk_ll_usec;
1463 		break;
1464 #endif
1465 
1466 	case SO_MAX_PACING_RATE:
1467 		if (sizeof(v.ulval) != sizeof(v.val) && len >= sizeof(v.ulval)) {
1468 			lv = sizeof(v.ulval);
1469 			v.ulval = sk->sk_max_pacing_rate;
1470 		} else {
1471 			/* 32bit version */
1472 			v.val = min_t(unsigned long, sk->sk_max_pacing_rate, ~0U);
1473 		}
1474 		break;
1475 
1476 	case SO_INCOMING_CPU:
1477 		v.val = READ_ONCE(sk->sk_incoming_cpu);
1478 		break;
1479 
1480 	case SO_MEMINFO:
1481 	{
1482 		u32 meminfo[SK_MEMINFO_VARS];
1483 
1484 		sk_get_meminfo(sk, meminfo);
1485 
1486 		len = min_t(unsigned int, len, sizeof(meminfo));
1487 		if (copy_to_user(optval, &meminfo, len))
1488 			return -EFAULT;
1489 
1490 		goto lenout;
1491 	}
1492 
1493 #ifdef CONFIG_NET_RX_BUSY_POLL
1494 	case SO_INCOMING_NAPI_ID:
1495 		v.val = READ_ONCE(sk->sk_napi_id);
1496 
1497 		/* aggregate non-NAPI IDs down to 0 */
1498 		if (v.val < MIN_NAPI_ID)
1499 			v.val = 0;
1500 
1501 		break;
1502 #endif
1503 
1504 	case SO_COOKIE:
1505 		lv = sizeof(u64);
1506 		if (len < lv)
1507 			return -EINVAL;
1508 		v.val64 = sock_gen_cookie(sk);
1509 		break;
1510 
1511 	case SO_ZEROCOPY:
1512 		v.val = sock_flag(sk, SOCK_ZEROCOPY);
1513 		break;
1514 
1515 	case SO_TXTIME:
1516 		lv = sizeof(v.txtime);
1517 		v.txtime.clockid = sk->sk_clockid;
1518 		v.txtime.flags |= sk->sk_txtime_deadline_mode ?
1519 				  SOF_TXTIME_DEADLINE_MODE : 0;
1520 		v.txtime.flags |= sk->sk_txtime_report_errors ?
1521 				  SOF_TXTIME_REPORT_ERRORS : 0;
1522 		break;
1523 
1524 	case SO_BINDTOIFINDEX:
1525 		v.val = sk->sk_bound_dev_if;
1526 		break;
1527 
1528 	default:
1529 		/* We implement the SO_SNDLOWAT etc to not be settable
1530 		 * (1003.1g 7).
1531 		 */
1532 		return -ENOPROTOOPT;
1533 	}
1534 
1535 	if (len > lv)
1536 		len = lv;
1537 	if (copy_to_user(optval, &v, len))
1538 		return -EFAULT;
1539 lenout:
1540 	if (put_user(len, optlen))
1541 		return -EFAULT;
1542 	return 0;
1543 }
1544 
1545 /*
1546  * Initialize an sk_lock.
1547  *
1548  * (We also register the sk_lock with the lock validator.)
1549  */
1550 static inline void sock_lock_init(struct sock *sk)
1551 {
1552 	if (sk->sk_kern_sock)
1553 		sock_lock_init_class_and_name(
1554 			sk,
1555 			af_family_kern_slock_key_strings[sk->sk_family],
1556 			af_family_kern_slock_keys + sk->sk_family,
1557 			af_family_kern_key_strings[sk->sk_family],
1558 			af_family_kern_keys + sk->sk_family);
1559 	else
1560 		sock_lock_init_class_and_name(
1561 			sk,
1562 			af_family_slock_key_strings[sk->sk_family],
1563 			af_family_slock_keys + sk->sk_family,
1564 			af_family_key_strings[sk->sk_family],
1565 			af_family_keys + sk->sk_family);
1566 }
1567 
1568 /*
1569  * Copy all fields from osk to nsk but nsk->sk_refcnt must not change yet,
1570  * even temporarly, because of RCU lookups. sk_node should also be left as is.
1571  * We must not copy fields between sk_dontcopy_begin and sk_dontcopy_end
1572  */
1573 static void sock_copy(struct sock *nsk, const struct sock *osk)
1574 {
1575 #ifdef CONFIG_SECURITY_NETWORK
1576 	void *sptr = nsk->sk_security;
1577 #endif
1578 	memcpy(nsk, osk, offsetof(struct sock, sk_dontcopy_begin));
1579 
1580 	memcpy(&nsk->sk_dontcopy_end, &osk->sk_dontcopy_end,
1581 	       osk->sk_prot->obj_size - offsetof(struct sock, sk_dontcopy_end));
1582 
1583 #ifdef CONFIG_SECURITY_NETWORK
1584 	nsk->sk_security = sptr;
1585 	security_sk_clone(osk, nsk);
1586 #endif
1587 }
1588 
1589 static struct sock *sk_prot_alloc(struct proto *prot, gfp_t priority,
1590 		int family)
1591 {
1592 	struct sock *sk;
1593 	struct kmem_cache *slab;
1594 
1595 	slab = prot->slab;
1596 	if (slab != NULL) {
1597 		sk = kmem_cache_alloc(slab, priority & ~__GFP_ZERO);
1598 		if (!sk)
1599 			return sk;
1600 		if (want_init_on_alloc(priority))
1601 			sk_prot_clear_nulls(sk, prot->obj_size);
1602 	} else
1603 		sk = kmalloc(prot->obj_size, priority);
1604 
1605 	if (sk != NULL) {
1606 		if (security_sk_alloc(sk, family, priority))
1607 			goto out_free;
1608 
1609 		if (!try_module_get(prot->owner))
1610 			goto out_free_sec;
1611 		sk_tx_queue_clear(sk);
1612 	}
1613 
1614 	return sk;
1615 
1616 out_free_sec:
1617 	security_sk_free(sk);
1618 out_free:
1619 	if (slab != NULL)
1620 		kmem_cache_free(slab, sk);
1621 	else
1622 		kfree(sk);
1623 	return NULL;
1624 }
1625 
1626 static void sk_prot_free(struct proto *prot, struct sock *sk)
1627 {
1628 	struct kmem_cache *slab;
1629 	struct module *owner;
1630 
1631 	owner = prot->owner;
1632 	slab = prot->slab;
1633 
1634 	cgroup_sk_free(&sk->sk_cgrp_data);
1635 	mem_cgroup_sk_free(sk);
1636 	security_sk_free(sk);
1637 	if (slab != NULL)
1638 		kmem_cache_free(slab, sk);
1639 	else
1640 		kfree(sk);
1641 	module_put(owner);
1642 }
1643 
1644 /**
1645  *	sk_alloc - All socket objects are allocated here
1646  *	@net: the applicable net namespace
1647  *	@family: protocol family
1648  *	@priority: for allocation (%GFP_KERNEL, %GFP_ATOMIC, etc)
1649  *	@prot: struct proto associated with this new sock instance
1650  *	@kern: is this to be a kernel socket?
1651  */
1652 struct sock *sk_alloc(struct net *net, int family, gfp_t priority,
1653 		      struct proto *prot, int kern)
1654 {
1655 	struct sock *sk;
1656 
1657 	sk = sk_prot_alloc(prot, priority | __GFP_ZERO, family);
1658 	if (sk) {
1659 		sk->sk_family = family;
1660 		/*
1661 		 * See comment in struct sock definition to understand
1662 		 * why we need sk_prot_creator -acme
1663 		 */
1664 		sk->sk_prot = sk->sk_prot_creator = prot;
1665 		sk->sk_kern_sock = kern;
1666 		sock_lock_init(sk);
1667 		sk->sk_net_refcnt = kern ? 0 : 1;
1668 		if (likely(sk->sk_net_refcnt)) {
1669 			get_net(net);
1670 			sock_inuse_add(net, 1);
1671 		}
1672 
1673 		sock_net_set(sk, net);
1674 		refcount_set(&sk->sk_wmem_alloc, 1);
1675 
1676 		mem_cgroup_sk_alloc(sk);
1677 		cgroup_sk_alloc(&sk->sk_cgrp_data);
1678 		sock_update_classid(&sk->sk_cgrp_data);
1679 		sock_update_netprioidx(&sk->sk_cgrp_data);
1680 	}
1681 
1682 	return sk;
1683 }
1684 EXPORT_SYMBOL(sk_alloc);
1685 
1686 /* Sockets having SOCK_RCU_FREE will call this function after one RCU
1687  * grace period. This is the case for UDP sockets and TCP listeners.
1688  */
1689 static void __sk_destruct(struct rcu_head *head)
1690 {
1691 	struct sock *sk = container_of(head, struct sock, sk_rcu);
1692 	struct sk_filter *filter;
1693 
1694 	if (sk->sk_destruct)
1695 		sk->sk_destruct(sk);
1696 
1697 	filter = rcu_dereference_check(sk->sk_filter,
1698 				       refcount_read(&sk->sk_wmem_alloc) == 0);
1699 	if (filter) {
1700 		sk_filter_uncharge(sk, filter);
1701 		RCU_INIT_POINTER(sk->sk_filter, NULL);
1702 	}
1703 
1704 	sock_disable_timestamp(sk, SK_FLAGS_TIMESTAMP);
1705 
1706 #ifdef CONFIG_BPF_SYSCALL
1707 	bpf_sk_storage_free(sk);
1708 #endif
1709 
1710 	if (atomic_read(&sk->sk_omem_alloc))
1711 		pr_debug("%s: optmem leakage (%d bytes) detected\n",
1712 			 __func__, atomic_read(&sk->sk_omem_alloc));
1713 
1714 	if (sk->sk_frag.page) {
1715 		put_page(sk->sk_frag.page);
1716 		sk->sk_frag.page = NULL;
1717 	}
1718 
1719 	if (sk->sk_peer_cred)
1720 		put_cred(sk->sk_peer_cred);
1721 	put_pid(sk->sk_peer_pid);
1722 	if (likely(sk->sk_net_refcnt))
1723 		put_net(sock_net(sk));
1724 	sk_prot_free(sk->sk_prot_creator, sk);
1725 }
1726 
1727 void sk_destruct(struct sock *sk)
1728 {
1729 	bool use_call_rcu = sock_flag(sk, SOCK_RCU_FREE);
1730 
1731 	if (rcu_access_pointer(sk->sk_reuseport_cb)) {
1732 		reuseport_detach_sock(sk);
1733 		use_call_rcu = true;
1734 	}
1735 
1736 	if (use_call_rcu)
1737 		call_rcu(&sk->sk_rcu, __sk_destruct);
1738 	else
1739 		__sk_destruct(&sk->sk_rcu);
1740 }
1741 
1742 static void __sk_free(struct sock *sk)
1743 {
1744 	if (likely(sk->sk_net_refcnt))
1745 		sock_inuse_add(sock_net(sk), -1);
1746 
1747 	if (unlikely(sk->sk_net_refcnt && sock_diag_has_destroy_listeners(sk)))
1748 		sock_diag_broadcast_destroy(sk);
1749 	else
1750 		sk_destruct(sk);
1751 }
1752 
1753 void sk_free(struct sock *sk)
1754 {
1755 	/*
1756 	 * We subtract one from sk_wmem_alloc and can know if
1757 	 * some packets are still in some tx queue.
1758 	 * If not null, sock_wfree() will call __sk_free(sk) later
1759 	 */
1760 	if (refcount_dec_and_test(&sk->sk_wmem_alloc))
1761 		__sk_free(sk);
1762 }
1763 EXPORT_SYMBOL(sk_free);
1764 
1765 static void sk_init_common(struct sock *sk)
1766 {
1767 	skb_queue_head_init(&sk->sk_receive_queue);
1768 	skb_queue_head_init(&sk->sk_write_queue);
1769 	skb_queue_head_init(&sk->sk_error_queue);
1770 
1771 	rwlock_init(&sk->sk_callback_lock);
1772 	lockdep_set_class_and_name(&sk->sk_receive_queue.lock,
1773 			af_rlock_keys + sk->sk_family,
1774 			af_family_rlock_key_strings[sk->sk_family]);
1775 	lockdep_set_class_and_name(&sk->sk_write_queue.lock,
1776 			af_wlock_keys + sk->sk_family,
1777 			af_family_wlock_key_strings[sk->sk_family]);
1778 	lockdep_set_class_and_name(&sk->sk_error_queue.lock,
1779 			af_elock_keys + sk->sk_family,
1780 			af_family_elock_key_strings[sk->sk_family]);
1781 	lockdep_set_class_and_name(&sk->sk_callback_lock,
1782 			af_callback_keys + sk->sk_family,
1783 			af_family_clock_key_strings[sk->sk_family]);
1784 }
1785 
1786 /**
1787  *	sk_clone_lock - clone a socket, and lock its clone
1788  *	@sk: the socket to clone
1789  *	@priority: for allocation (%GFP_KERNEL, %GFP_ATOMIC, etc)
1790  *
1791  *	Caller must unlock socket even in error path (bh_unlock_sock(newsk))
1792  */
1793 struct sock *sk_clone_lock(const struct sock *sk, const gfp_t priority)
1794 {
1795 	struct sock *newsk;
1796 	bool is_charged = true;
1797 
1798 	newsk = sk_prot_alloc(sk->sk_prot, priority, sk->sk_family);
1799 	if (newsk != NULL) {
1800 		struct sk_filter *filter;
1801 
1802 		sock_copy(newsk, sk);
1803 
1804 		newsk->sk_prot_creator = sk->sk_prot;
1805 
1806 		/* SANITY */
1807 		if (likely(newsk->sk_net_refcnt))
1808 			get_net(sock_net(newsk));
1809 		sk_node_init(&newsk->sk_node);
1810 		sock_lock_init(newsk);
1811 		bh_lock_sock(newsk);
1812 		newsk->sk_backlog.head	= newsk->sk_backlog.tail = NULL;
1813 		newsk->sk_backlog.len = 0;
1814 
1815 		atomic_set(&newsk->sk_rmem_alloc, 0);
1816 		/*
1817 		 * sk_wmem_alloc set to one (see sk_free() and sock_wfree())
1818 		 */
1819 		refcount_set(&newsk->sk_wmem_alloc, 1);
1820 		atomic_set(&newsk->sk_omem_alloc, 0);
1821 		sk_init_common(newsk);
1822 
1823 		newsk->sk_dst_cache	= NULL;
1824 		newsk->sk_dst_pending_confirm = 0;
1825 		newsk->sk_wmem_queued	= 0;
1826 		newsk->sk_forward_alloc = 0;
1827 		atomic_set(&newsk->sk_drops, 0);
1828 		newsk->sk_send_head	= NULL;
1829 		newsk->sk_userlocks	= sk->sk_userlocks & ~SOCK_BINDPORT_LOCK;
1830 		atomic_set(&newsk->sk_zckey, 0);
1831 
1832 		sock_reset_flag(newsk, SOCK_DONE);
1833 
1834 		/* sk->sk_memcg will be populated at accept() time */
1835 		newsk->sk_memcg = NULL;
1836 
1837 		cgroup_sk_alloc(&newsk->sk_cgrp_data);
1838 
1839 		rcu_read_lock();
1840 		filter = rcu_dereference(sk->sk_filter);
1841 		if (filter != NULL)
1842 			/* though it's an empty new sock, the charging may fail
1843 			 * if sysctl_optmem_max was changed between creation of
1844 			 * original socket and cloning
1845 			 */
1846 			is_charged = sk_filter_charge(newsk, filter);
1847 		RCU_INIT_POINTER(newsk->sk_filter, filter);
1848 		rcu_read_unlock();
1849 
1850 		if (unlikely(!is_charged || xfrm_sk_clone_policy(newsk, sk))) {
1851 			/* We need to make sure that we don't uncharge the new
1852 			 * socket if we couldn't charge it in the first place
1853 			 * as otherwise we uncharge the parent's filter.
1854 			 */
1855 			if (!is_charged)
1856 				RCU_INIT_POINTER(newsk->sk_filter, NULL);
1857 			sk_free_unlock_clone(newsk);
1858 			newsk = NULL;
1859 			goto out;
1860 		}
1861 		RCU_INIT_POINTER(newsk->sk_reuseport_cb, NULL);
1862 
1863 		if (bpf_sk_storage_clone(sk, newsk)) {
1864 			sk_free_unlock_clone(newsk);
1865 			newsk = NULL;
1866 			goto out;
1867 		}
1868 
1869 		newsk->sk_err	   = 0;
1870 		newsk->sk_err_soft = 0;
1871 		newsk->sk_priority = 0;
1872 		newsk->sk_incoming_cpu = raw_smp_processor_id();
1873 		if (likely(newsk->sk_net_refcnt))
1874 			sock_inuse_add(sock_net(newsk), 1);
1875 
1876 		/*
1877 		 * Before updating sk_refcnt, we must commit prior changes to memory
1878 		 * (Documentation/RCU/rculist_nulls.txt for details)
1879 		 */
1880 		smp_wmb();
1881 		refcount_set(&newsk->sk_refcnt, 2);
1882 
1883 		/*
1884 		 * Increment the counter in the same struct proto as the master
1885 		 * sock (sk_refcnt_debug_inc uses newsk->sk_prot->socks, that
1886 		 * is the same as sk->sk_prot->socks, as this field was copied
1887 		 * with memcpy).
1888 		 *
1889 		 * This _changes_ the previous behaviour, where
1890 		 * tcp_create_openreq_child always was incrementing the
1891 		 * equivalent to tcp_prot->socks (inet_sock_nr), so this have
1892 		 * to be taken into account in all callers. -acme
1893 		 */
1894 		sk_refcnt_debug_inc(newsk);
1895 		sk_set_socket(newsk, NULL);
1896 		RCU_INIT_POINTER(newsk->sk_wq, NULL);
1897 
1898 		if (newsk->sk_prot->sockets_allocated)
1899 			sk_sockets_allocated_inc(newsk);
1900 
1901 		if (sock_needs_netstamp(sk) &&
1902 		    newsk->sk_flags & SK_FLAGS_TIMESTAMP)
1903 			net_enable_timestamp();
1904 	}
1905 out:
1906 	return newsk;
1907 }
1908 EXPORT_SYMBOL_GPL(sk_clone_lock);
1909 
1910 void sk_free_unlock_clone(struct sock *sk)
1911 {
1912 	/* It is still raw copy of parent, so invalidate
1913 	 * destructor and make plain sk_free() */
1914 	sk->sk_destruct = NULL;
1915 	bh_unlock_sock(sk);
1916 	sk_free(sk);
1917 }
1918 EXPORT_SYMBOL_GPL(sk_free_unlock_clone);
1919 
1920 void sk_setup_caps(struct sock *sk, struct dst_entry *dst)
1921 {
1922 	u32 max_segs = 1;
1923 
1924 	sk_dst_set(sk, dst);
1925 	sk->sk_route_caps = dst->dev->features | sk->sk_route_forced_caps;
1926 	if (sk->sk_route_caps & NETIF_F_GSO)
1927 		sk->sk_route_caps |= NETIF_F_GSO_SOFTWARE;
1928 	sk->sk_route_caps &= ~sk->sk_route_nocaps;
1929 	if (sk_can_gso(sk)) {
1930 		if (dst->header_len && !xfrm_dst_offload_ok(dst)) {
1931 			sk->sk_route_caps &= ~NETIF_F_GSO_MASK;
1932 		} else {
1933 			sk->sk_route_caps |= NETIF_F_SG | NETIF_F_HW_CSUM;
1934 			sk->sk_gso_max_size = dst->dev->gso_max_size;
1935 			max_segs = max_t(u32, dst->dev->gso_max_segs, 1);
1936 		}
1937 	}
1938 	sk->sk_gso_max_segs = max_segs;
1939 }
1940 EXPORT_SYMBOL_GPL(sk_setup_caps);
1941 
1942 /*
1943  *	Simple resource managers for sockets.
1944  */
1945 
1946 
1947 /*
1948  * Write buffer destructor automatically called from kfree_skb.
1949  */
1950 void sock_wfree(struct sk_buff *skb)
1951 {
1952 	struct sock *sk = skb->sk;
1953 	unsigned int len = skb->truesize;
1954 
1955 	if (!sock_flag(sk, SOCK_USE_WRITE_QUEUE)) {
1956 		/*
1957 		 * Keep a reference on sk_wmem_alloc, this will be released
1958 		 * after sk_write_space() call
1959 		 */
1960 		WARN_ON(refcount_sub_and_test(len - 1, &sk->sk_wmem_alloc));
1961 		sk->sk_write_space(sk);
1962 		len = 1;
1963 	}
1964 	/*
1965 	 * if sk_wmem_alloc reaches 0, we must finish what sk_free()
1966 	 * could not do because of in-flight packets
1967 	 */
1968 	if (refcount_sub_and_test(len, &sk->sk_wmem_alloc))
1969 		__sk_free(sk);
1970 }
1971 EXPORT_SYMBOL(sock_wfree);
1972 
1973 /* This variant of sock_wfree() is used by TCP,
1974  * since it sets SOCK_USE_WRITE_QUEUE.
1975  */
1976 void __sock_wfree(struct sk_buff *skb)
1977 {
1978 	struct sock *sk = skb->sk;
1979 
1980 	if (refcount_sub_and_test(skb->truesize, &sk->sk_wmem_alloc))
1981 		__sk_free(sk);
1982 }
1983 
1984 void skb_set_owner_w(struct sk_buff *skb, struct sock *sk)
1985 {
1986 	skb_orphan(skb);
1987 	skb->sk = sk;
1988 #ifdef CONFIG_INET
1989 	if (unlikely(!sk_fullsock(sk))) {
1990 		skb->destructor = sock_edemux;
1991 		sock_hold(sk);
1992 		return;
1993 	}
1994 #endif
1995 	skb->destructor = sock_wfree;
1996 	skb_set_hash_from_sk(skb, sk);
1997 	/*
1998 	 * We used to take a refcount on sk, but following operation
1999 	 * is enough to guarantee sk_free() wont free this sock until
2000 	 * all in-flight packets are completed
2001 	 */
2002 	refcount_add(skb->truesize, &sk->sk_wmem_alloc);
2003 }
2004 EXPORT_SYMBOL(skb_set_owner_w);
2005 
2006 static bool can_skb_orphan_partial(const struct sk_buff *skb)
2007 {
2008 #ifdef CONFIG_TLS_DEVICE
2009 	/* Drivers depend on in-order delivery for crypto offload,
2010 	 * partial orphan breaks out-of-order-OK logic.
2011 	 */
2012 	if (skb->decrypted)
2013 		return false;
2014 #endif
2015 	return (skb->destructor == sock_wfree ||
2016 		(IS_ENABLED(CONFIG_INET) && skb->destructor == tcp_wfree));
2017 }
2018 
2019 /* This helper is used by netem, as it can hold packets in its
2020  * delay queue. We want to allow the owner socket to send more
2021  * packets, as if they were already TX completed by a typical driver.
2022  * But we also want to keep skb->sk set because some packet schedulers
2023  * rely on it (sch_fq for example).
2024  */
2025 void skb_orphan_partial(struct sk_buff *skb)
2026 {
2027 	if (skb_is_tcp_pure_ack(skb))
2028 		return;
2029 
2030 	if (can_skb_orphan_partial(skb)) {
2031 		struct sock *sk = skb->sk;
2032 
2033 		if (refcount_inc_not_zero(&sk->sk_refcnt)) {
2034 			WARN_ON(refcount_sub_and_test(skb->truesize, &sk->sk_wmem_alloc));
2035 			skb->destructor = sock_efree;
2036 		}
2037 	} else {
2038 		skb_orphan(skb);
2039 	}
2040 }
2041 EXPORT_SYMBOL(skb_orphan_partial);
2042 
2043 /*
2044  * Read buffer destructor automatically called from kfree_skb.
2045  */
2046 void sock_rfree(struct sk_buff *skb)
2047 {
2048 	struct sock *sk = skb->sk;
2049 	unsigned int len = skb->truesize;
2050 
2051 	atomic_sub(len, &sk->sk_rmem_alloc);
2052 	sk_mem_uncharge(sk, len);
2053 }
2054 EXPORT_SYMBOL(sock_rfree);
2055 
2056 /*
2057  * Buffer destructor for skbs that are not used directly in read or write
2058  * path, e.g. for error handler skbs. Automatically called from kfree_skb.
2059  */
2060 void sock_efree(struct sk_buff *skb)
2061 {
2062 	sock_put(skb->sk);
2063 }
2064 EXPORT_SYMBOL(sock_efree);
2065 
2066 kuid_t sock_i_uid(struct sock *sk)
2067 {
2068 	kuid_t uid;
2069 
2070 	read_lock_bh(&sk->sk_callback_lock);
2071 	uid = sk->sk_socket ? SOCK_INODE(sk->sk_socket)->i_uid : GLOBAL_ROOT_UID;
2072 	read_unlock_bh(&sk->sk_callback_lock);
2073 	return uid;
2074 }
2075 EXPORT_SYMBOL(sock_i_uid);
2076 
2077 unsigned long sock_i_ino(struct sock *sk)
2078 {
2079 	unsigned long ino;
2080 
2081 	read_lock_bh(&sk->sk_callback_lock);
2082 	ino = sk->sk_socket ? SOCK_INODE(sk->sk_socket)->i_ino : 0;
2083 	read_unlock_bh(&sk->sk_callback_lock);
2084 	return ino;
2085 }
2086 EXPORT_SYMBOL(sock_i_ino);
2087 
2088 /*
2089  * Allocate a skb from the socket's send buffer.
2090  */
2091 struct sk_buff *sock_wmalloc(struct sock *sk, unsigned long size, int force,
2092 			     gfp_t priority)
2093 {
2094 	if (force ||
2095 	    refcount_read(&sk->sk_wmem_alloc) < READ_ONCE(sk->sk_sndbuf)) {
2096 		struct sk_buff *skb = alloc_skb(size, priority);
2097 
2098 		if (skb) {
2099 			skb_set_owner_w(skb, sk);
2100 			return skb;
2101 		}
2102 	}
2103 	return NULL;
2104 }
2105 EXPORT_SYMBOL(sock_wmalloc);
2106 
2107 static void sock_ofree(struct sk_buff *skb)
2108 {
2109 	struct sock *sk = skb->sk;
2110 
2111 	atomic_sub(skb->truesize, &sk->sk_omem_alloc);
2112 }
2113 
2114 struct sk_buff *sock_omalloc(struct sock *sk, unsigned long size,
2115 			     gfp_t priority)
2116 {
2117 	struct sk_buff *skb;
2118 
2119 	/* small safe race: SKB_TRUESIZE may differ from final skb->truesize */
2120 	if (atomic_read(&sk->sk_omem_alloc) + SKB_TRUESIZE(size) >
2121 	    sysctl_optmem_max)
2122 		return NULL;
2123 
2124 	skb = alloc_skb(size, priority);
2125 	if (!skb)
2126 		return NULL;
2127 
2128 	atomic_add(skb->truesize, &sk->sk_omem_alloc);
2129 	skb->sk = sk;
2130 	skb->destructor = sock_ofree;
2131 	return skb;
2132 }
2133 
2134 /*
2135  * Allocate a memory block from the socket's option memory buffer.
2136  */
2137 void *sock_kmalloc(struct sock *sk, int size, gfp_t priority)
2138 {
2139 	if ((unsigned int)size <= sysctl_optmem_max &&
2140 	    atomic_read(&sk->sk_omem_alloc) + size < sysctl_optmem_max) {
2141 		void *mem;
2142 		/* First do the add, to avoid the race if kmalloc
2143 		 * might sleep.
2144 		 */
2145 		atomic_add(size, &sk->sk_omem_alloc);
2146 		mem = kmalloc(size, priority);
2147 		if (mem)
2148 			return mem;
2149 		atomic_sub(size, &sk->sk_omem_alloc);
2150 	}
2151 	return NULL;
2152 }
2153 EXPORT_SYMBOL(sock_kmalloc);
2154 
2155 /* Free an option memory block. Note, we actually want the inline
2156  * here as this allows gcc to detect the nullify and fold away the
2157  * condition entirely.
2158  */
2159 static inline void __sock_kfree_s(struct sock *sk, void *mem, int size,
2160 				  const bool nullify)
2161 {
2162 	if (WARN_ON_ONCE(!mem))
2163 		return;
2164 	if (nullify)
2165 		kzfree(mem);
2166 	else
2167 		kfree(mem);
2168 	atomic_sub(size, &sk->sk_omem_alloc);
2169 }
2170 
2171 void sock_kfree_s(struct sock *sk, void *mem, int size)
2172 {
2173 	__sock_kfree_s(sk, mem, size, false);
2174 }
2175 EXPORT_SYMBOL(sock_kfree_s);
2176 
2177 void sock_kzfree_s(struct sock *sk, void *mem, int size)
2178 {
2179 	__sock_kfree_s(sk, mem, size, true);
2180 }
2181 EXPORT_SYMBOL(sock_kzfree_s);
2182 
2183 /* It is almost wait_for_tcp_memory minus release_sock/lock_sock.
2184    I think, these locks should be removed for datagram sockets.
2185  */
2186 static long sock_wait_for_wmem(struct sock *sk, long timeo)
2187 {
2188 	DEFINE_WAIT(wait);
2189 
2190 	sk_clear_bit(SOCKWQ_ASYNC_NOSPACE, sk);
2191 	for (;;) {
2192 		if (!timeo)
2193 			break;
2194 		if (signal_pending(current))
2195 			break;
2196 		set_bit(SOCK_NOSPACE, &sk->sk_socket->flags);
2197 		prepare_to_wait(sk_sleep(sk), &wait, TASK_INTERRUPTIBLE);
2198 		if (refcount_read(&sk->sk_wmem_alloc) < READ_ONCE(sk->sk_sndbuf))
2199 			break;
2200 		if (sk->sk_shutdown & SEND_SHUTDOWN)
2201 			break;
2202 		if (sk->sk_err)
2203 			break;
2204 		timeo = schedule_timeout(timeo);
2205 	}
2206 	finish_wait(sk_sleep(sk), &wait);
2207 	return timeo;
2208 }
2209 
2210 
2211 /*
2212  *	Generic send/receive buffer handlers
2213  */
2214 
2215 struct sk_buff *sock_alloc_send_pskb(struct sock *sk, unsigned long header_len,
2216 				     unsigned long data_len, int noblock,
2217 				     int *errcode, int max_page_order)
2218 {
2219 	struct sk_buff *skb;
2220 	long timeo;
2221 	int err;
2222 
2223 	timeo = sock_sndtimeo(sk, noblock);
2224 	for (;;) {
2225 		err = sock_error(sk);
2226 		if (err != 0)
2227 			goto failure;
2228 
2229 		err = -EPIPE;
2230 		if (sk->sk_shutdown & SEND_SHUTDOWN)
2231 			goto failure;
2232 
2233 		if (sk_wmem_alloc_get(sk) < READ_ONCE(sk->sk_sndbuf))
2234 			break;
2235 
2236 		sk_set_bit(SOCKWQ_ASYNC_NOSPACE, sk);
2237 		set_bit(SOCK_NOSPACE, &sk->sk_socket->flags);
2238 		err = -EAGAIN;
2239 		if (!timeo)
2240 			goto failure;
2241 		if (signal_pending(current))
2242 			goto interrupted;
2243 		timeo = sock_wait_for_wmem(sk, timeo);
2244 	}
2245 	skb = alloc_skb_with_frags(header_len, data_len, max_page_order,
2246 				   errcode, sk->sk_allocation);
2247 	if (skb)
2248 		skb_set_owner_w(skb, sk);
2249 	return skb;
2250 
2251 interrupted:
2252 	err = sock_intr_errno(timeo);
2253 failure:
2254 	*errcode = err;
2255 	return NULL;
2256 }
2257 EXPORT_SYMBOL(sock_alloc_send_pskb);
2258 
2259 struct sk_buff *sock_alloc_send_skb(struct sock *sk, unsigned long size,
2260 				    int noblock, int *errcode)
2261 {
2262 	return sock_alloc_send_pskb(sk, size, 0, noblock, errcode, 0);
2263 }
2264 EXPORT_SYMBOL(sock_alloc_send_skb);
2265 
2266 int __sock_cmsg_send(struct sock *sk, struct msghdr *msg, struct cmsghdr *cmsg,
2267 		     struct sockcm_cookie *sockc)
2268 {
2269 	u32 tsflags;
2270 
2271 	switch (cmsg->cmsg_type) {
2272 	case SO_MARK:
2273 		if (!ns_capable(sock_net(sk)->user_ns, CAP_NET_ADMIN))
2274 			return -EPERM;
2275 		if (cmsg->cmsg_len != CMSG_LEN(sizeof(u32)))
2276 			return -EINVAL;
2277 		sockc->mark = *(u32 *)CMSG_DATA(cmsg);
2278 		break;
2279 	case SO_TIMESTAMPING_OLD:
2280 		if (cmsg->cmsg_len != CMSG_LEN(sizeof(u32)))
2281 			return -EINVAL;
2282 
2283 		tsflags = *(u32 *)CMSG_DATA(cmsg);
2284 		if (tsflags & ~SOF_TIMESTAMPING_TX_RECORD_MASK)
2285 			return -EINVAL;
2286 
2287 		sockc->tsflags &= ~SOF_TIMESTAMPING_TX_RECORD_MASK;
2288 		sockc->tsflags |= tsflags;
2289 		break;
2290 	case SCM_TXTIME:
2291 		if (!sock_flag(sk, SOCK_TXTIME))
2292 			return -EINVAL;
2293 		if (cmsg->cmsg_len != CMSG_LEN(sizeof(u64)))
2294 			return -EINVAL;
2295 		sockc->transmit_time = get_unaligned((u64 *)CMSG_DATA(cmsg));
2296 		break;
2297 	/* SCM_RIGHTS and SCM_CREDENTIALS are semantically in SOL_UNIX. */
2298 	case SCM_RIGHTS:
2299 	case SCM_CREDENTIALS:
2300 		break;
2301 	default:
2302 		return -EINVAL;
2303 	}
2304 	return 0;
2305 }
2306 EXPORT_SYMBOL(__sock_cmsg_send);
2307 
2308 int sock_cmsg_send(struct sock *sk, struct msghdr *msg,
2309 		   struct sockcm_cookie *sockc)
2310 {
2311 	struct cmsghdr *cmsg;
2312 	int ret;
2313 
2314 	for_each_cmsghdr(cmsg, msg) {
2315 		if (!CMSG_OK(msg, cmsg))
2316 			return -EINVAL;
2317 		if (cmsg->cmsg_level != SOL_SOCKET)
2318 			continue;
2319 		ret = __sock_cmsg_send(sk, msg, cmsg, sockc);
2320 		if (ret)
2321 			return ret;
2322 	}
2323 	return 0;
2324 }
2325 EXPORT_SYMBOL(sock_cmsg_send);
2326 
2327 static void sk_enter_memory_pressure(struct sock *sk)
2328 {
2329 	if (!sk->sk_prot->enter_memory_pressure)
2330 		return;
2331 
2332 	sk->sk_prot->enter_memory_pressure(sk);
2333 }
2334 
2335 static void sk_leave_memory_pressure(struct sock *sk)
2336 {
2337 	if (sk->sk_prot->leave_memory_pressure) {
2338 		sk->sk_prot->leave_memory_pressure(sk);
2339 	} else {
2340 		unsigned long *memory_pressure = sk->sk_prot->memory_pressure;
2341 
2342 		if (memory_pressure && READ_ONCE(*memory_pressure))
2343 			WRITE_ONCE(*memory_pressure, 0);
2344 	}
2345 }
2346 
2347 /* On 32bit arches, an skb frag is limited to 2^15 */
2348 #define SKB_FRAG_PAGE_ORDER	get_order(32768)
2349 DEFINE_STATIC_KEY_FALSE(net_high_order_alloc_disable_key);
2350 
2351 /**
2352  * skb_page_frag_refill - check that a page_frag contains enough room
2353  * @sz: minimum size of the fragment we want to get
2354  * @pfrag: pointer to page_frag
2355  * @gfp: priority for memory allocation
2356  *
2357  * Note: While this allocator tries to use high order pages, there is
2358  * no guarantee that allocations succeed. Therefore, @sz MUST be
2359  * less or equal than PAGE_SIZE.
2360  */
2361 bool skb_page_frag_refill(unsigned int sz, struct page_frag *pfrag, gfp_t gfp)
2362 {
2363 	if (pfrag->page) {
2364 		if (page_ref_count(pfrag->page) == 1) {
2365 			pfrag->offset = 0;
2366 			return true;
2367 		}
2368 		if (pfrag->offset + sz <= pfrag->size)
2369 			return true;
2370 		put_page(pfrag->page);
2371 	}
2372 
2373 	pfrag->offset = 0;
2374 	if (SKB_FRAG_PAGE_ORDER &&
2375 	    !static_branch_unlikely(&net_high_order_alloc_disable_key)) {
2376 		/* Avoid direct reclaim but allow kswapd to wake */
2377 		pfrag->page = alloc_pages((gfp & ~__GFP_DIRECT_RECLAIM) |
2378 					  __GFP_COMP | __GFP_NOWARN |
2379 					  __GFP_NORETRY,
2380 					  SKB_FRAG_PAGE_ORDER);
2381 		if (likely(pfrag->page)) {
2382 			pfrag->size = PAGE_SIZE << SKB_FRAG_PAGE_ORDER;
2383 			return true;
2384 		}
2385 	}
2386 	pfrag->page = alloc_page(gfp);
2387 	if (likely(pfrag->page)) {
2388 		pfrag->size = PAGE_SIZE;
2389 		return true;
2390 	}
2391 	return false;
2392 }
2393 EXPORT_SYMBOL(skb_page_frag_refill);
2394 
2395 bool sk_page_frag_refill(struct sock *sk, struct page_frag *pfrag)
2396 {
2397 	if (likely(skb_page_frag_refill(32U, pfrag, sk->sk_allocation)))
2398 		return true;
2399 
2400 	sk_enter_memory_pressure(sk);
2401 	sk_stream_moderate_sndbuf(sk);
2402 	return false;
2403 }
2404 EXPORT_SYMBOL(sk_page_frag_refill);
2405 
2406 static void __lock_sock(struct sock *sk)
2407 	__releases(&sk->sk_lock.slock)
2408 	__acquires(&sk->sk_lock.slock)
2409 {
2410 	DEFINE_WAIT(wait);
2411 
2412 	for (;;) {
2413 		prepare_to_wait_exclusive(&sk->sk_lock.wq, &wait,
2414 					TASK_UNINTERRUPTIBLE);
2415 		spin_unlock_bh(&sk->sk_lock.slock);
2416 		schedule();
2417 		spin_lock_bh(&sk->sk_lock.slock);
2418 		if (!sock_owned_by_user(sk))
2419 			break;
2420 	}
2421 	finish_wait(&sk->sk_lock.wq, &wait);
2422 }
2423 
2424 void __release_sock(struct sock *sk)
2425 	__releases(&sk->sk_lock.slock)
2426 	__acquires(&sk->sk_lock.slock)
2427 {
2428 	struct sk_buff *skb, *next;
2429 
2430 	while ((skb = sk->sk_backlog.head) != NULL) {
2431 		sk->sk_backlog.head = sk->sk_backlog.tail = NULL;
2432 
2433 		spin_unlock_bh(&sk->sk_lock.slock);
2434 
2435 		do {
2436 			next = skb->next;
2437 			prefetch(next);
2438 			WARN_ON_ONCE(skb_dst_is_noref(skb));
2439 			skb_mark_not_on_list(skb);
2440 			sk_backlog_rcv(sk, skb);
2441 
2442 			cond_resched();
2443 
2444 			skb = next;
2445 		} while (skb != NULL);
2446 
2447 		spin_lock_bh(&sk->sk_lock.slock);
2448 	}
2449 
2450 	/*
2451 	 * Doing the zeroing here guarantee we can not loop forever
2452 	 * while a wild producer attempts to flood us.
2453 	 */
2454 	sk->sk_backlog.len = 0;
2455 }
2456 
2457 void __sk_flush_backlog(struct sock *sk)
2458 {
2459 	spin_lock_bh(&sk->sk_lock.slock);
2460 	__release_sock(sk);
2461 	spin_unlock_bh(&sk->sk_lock.slock);
2462 }
2463 
2464 /**
2465  * sk_wait_data - wait for data to arrive at sk_receive_queue
2466  * @sk:    sock to wait on
2467  * @timeo: for how long
2468  * @skb:   last skb seen on sk_receive_queue
2469  *
2470  * Now socket state including sk->sk_err is changed only under lock,
2471  * hence we may omit checks after joining wait queue.
2472  * We check receive queue before schedule() only as optimization;
2473  * it is very likely that release_sock() added new data.
2474  */
2475 int sk_wait_data(struct sock *sk, long *timeo, const struct sk_buff *skb)
2476 {
2477 	DEFINE_WAIT_FUNC(wait, woken_wake_function);
2478 	int rc;
2479 
2480 	add_wait_queue(sk_sleep(sk), &wait);
2481 	sk_set_bit(SOCKWQ_ASYNC_WAITDATA, sk);
2482 	rc = sk_wait_event(sk, timeo, skb_peek_tail(&sk->sk_receive_queue) != skb, &wait);
2483 	sk_clear_bit(SOCKWQ_ASYNC_WAITDATA, sk);
2484 	remove_wait_queue(sk_sleep(sk), &wait);
2485 	return rc;
2486 }
2487 EXPORT_SYMBOL(sk_wait_data);
2488 
2489 /**
2490  *	__sk_mem_raise_allocated - increase memory_allocated
2491  *	@sk: socket
2492  *	@size: memory size to allocate
2493  *	@amt: pages to allocate
2494  *	@kind: allocation type
2495  *
2496  *	Similar to __sk_mem_schedule(), but does not update sk_forward_alloc
2497  */
2498 int __sk_mem_raise_allocated(struct sock *sk, int size, int amt, int kind)
2499 {
2500 	struct proto *prot = sk->sk_prot;
2501 	long allocated = sk_memory_allocated_add(sk, amt);
2502 	bool charged = true;
2503 
2504 	if (mem_cgroup_sockets_enabled && sk->sk_memcg &&
2505 	    !(charged = mem_cgroup_charge_skmem(sk->sk_memcg, amt)))
2506 		goto suppress_allocation;
2507 
2508 	/* Under limit. */
2509 	if (allocated <= sk_prot_mem_limits(sk, 0)) {
2510 		sk_leave_memory_pressure(sk);
2511 		return 1;
2512 	}
2513 
2514 	/* Under pressure. */
2515 	if (allocated > sk_prot_mem_limits(sk, 1))
2516 		sk_enter_memory_pressure(sk);
2517 
2518 	/* Over hard limit. */
2519 	if (allocated > sk_prot_mem_limits(sk, 2))
2520 		goto suppress_allocation;
2521 
2522 	/* guarantee minimum buffer size under pressure */
2523 	if (kind == SK_MEM_RECV) {
2524 		if (atomic_read(&sk->sk_rmem_alloc) < sk_get_rmem0(sk, prot))
2525 			return 1;
2526 
2527 	} else { /* SK_MEM_SEND */
2528 		int wmem0 = sk_get_wmem0(sk, prot);
2529 
2530 		if (sk->sk_type == SOCK_STREAM) {
2531 			if (sk->sk_wmem_queued < wmem0)
2532 				return 1;
2533 		} else if (refcount_read(&sk->sk_wmem_alloc) < wmem0) {
2534 				return 1;
2535 		}
2536 	}
2537 
2538 	if (sk_has_memory_pressure(sk)) {
2539 		u64 alloc;
2540 
2541 		if (!sk_under_memory_pressure(sk))
2542 			return 1;
2543 		alloc = sk_sockets_allocated_read_positive(sk);
2544 		if (sk_prot_mem_limits(sk, 2) > alloc *
2545 		    sk_mem_pages(sk->sk_wmem_queued +
2546 				 atomic_read(&sk->sk_rmem_alloc) +
2547 				 sk->sk_forward_alloc))
2548 			return 1;
2549 	}
2550 
2551 suppress_allocation:
2552 
2553 	if (kind == SK_MEM_SEND && sk->sk_type == SOCK_STREAM) {
2554 		sk_stream_moderate_sndbuf(sk);
2555 
2556 		/* Fail only if socket is _under_ its sndbuf.
2557 		 * In this case we cannot block, so that we have to fail.
2558 		 */
2559 		if (sk->sk_wmem_queued + size >= sk->sk_sndbuf)
2560 			return 1;
2561 	}
2562 
2563 	if (kind == SK_MEM_SEND || (kind == SK_MEM_RECV && charged))
2564 		trace_sock_exceed_buf_limit(sk, prot, allocated, kind);
2565 
2566 	sk_memory_allocated_sub(sk, amt);
2567 
2568 	if (mem_cgroup_sockets_enabled && sk->sk_memcg)
2569 		mem_cgroup_uncharge_skmem(sk->sk_memcg, amt);
2570 
2571 	return 0;
2572 }
2573 EXPORT_SYMBOL(__sk_mem_raise_allocated);
2574 
2575 /**
2576  *	__sk_mem_schedule - increase sk_forward_alloc and memory_allocated
2577  *	@sk: socket
2578  *	@size: memory size to allocate
2579  *	@kind: allocation type
2580  *
2581  *	If kind is SK_MEM_SEND, it means wmem allocation. Otherwise it means
2582  *	rmem allocation. This function assumes that protocols which have
2583  *	memory_pressure use sk_wmem_queued as write buffer accounting.
2584  */
2585 int __sk_mem_schedule(struct sock *sk, int size, int kind)
2586 {
2587 	int ret, amt = sk_mem_pages(size);
2588 
2589 	sk->sk_forward_alloc += amt << SK_MEM_QUANTUM_SHIFT;
2590 	ret = __sk_mem_raise_allocated(sk, size, amt, kind);
2591 	if (!ret)
2592 		sk->sk_forward_alloc -= amt << SK_MEM_QUANTUM_SHIFT;
2593 	return ret;
2594 }
2595 EXPORT_SYMBOL(__sk_mem_schedule);
2596 
2597 /**
2598  *	__sk_mem_reduce_allocated - reclaim memory_allocated
2599  *	@sk: socket
2600  *	@amount: number of quanta
2601  *
2602  *	Similar to __sk_mem_reclaim(), but does not update sk_forward_alloc
2603  */
2604 void __sk_mem_reduce_allocated(struct sock *sk, int amount)
2605 {
2606 	sk_memory_allocated_sub(sk, amount);
2607 
2608 	if (mem_cgroup_sockets_enabled && sk->sk_memcg)
2609 		mem_cgroup_uncharge_skmem(sk->sk_memcg, amount);
2610 
2611 	if (sk_under_memory_pressure(sk) &&
2612 	    (sk_memory_allocated(sk) < sk_prot_mem_limits(sk, 0)))
2613 		sk_leave_memory_pressure(sk);
2614 }
2615 EXPORT_SYMBOL(__sk_mem_reduce_allocated);
2616 
2617 /**
2618  *	__sk_mem_reclaim - reclaim sk_forward_alloc and memory_allocated
2619  *	@sk: socket
2620  *	@amount: number of bytes (rounded down to a SK_MEM_QUANTUM multiple)
2621  */
2622 void __sk_mem_reclaim(struct sock *sk, int amount)
2623 {
2624 	amount >>= SK_MEM_QUANTUM_SHIFT;
2625 	sk->sk_forward_alloc -= amount << SK_MEM_QUANTUM_SHIFT;
2626 	__sk_mem_reduce_allocated(sk, amount);
2627 }
2628 EXPORT_SYMBOL(__sk_mem_reclaim);
2629 
2630 int sk_set_peek_off(struct sock *sk, int val)
2631 {
2632 	sk->sk_peek_off = val;
2633 	return 0;
2634 }
2635 EXPORT_SYMBOL_GPL(sk_set_peek_off);
2636 
2637 /*
2638  * Set of default routines for initialising struct proto_ops when
2639  * the protocol does not support a particular function. In certain
2640  * cases where it makes no sense for a protocol to have a "do nothing"
2641  * function, some default processing is provided.
2642  */
2643 
2644 int sock_no_bind(struct socket *sock, struct sockaddr *saddr, int len)
2645 {
2646 	return -EOPNOTSUPP;
2647 }
2648 EXPORT_SYMBOL(sock_no_bind);
2649 
2650 int sock_no_connect(struct socket *sock, struct sockaddr *saddr,
2651 		    int len, int flags)
2652 {
2653 	return -EOPNOTSUPP;
2654 }
2655 EXPORT_SYMBOL(sock_no_connect);
2656 
2657 int sock_no_socketpair(struct socket *sock1, struct socket *sock2)
2658 {
2659 	return -EOPNOTSUPP;
2660 }
2661 EXPORT_SYMBOL(sock_no_socketpair);
2662 
2663 int sock_no_accept(struct socket *sock, struct socket *newsock, int flags,
2664 		   bool kern)
2665 {
2666 	return -EOPNOTSUPP;
2667 }
2668 EXPORT_SYMBOL(sock_no_accept);
2669 
2670 int sock_no_getname(struct socket *sock, struct sockaddr *saddr,
2671 		    int peer)
2672 {
2673 	return -EOPNOTSUPP;
2674 }
2675 EXPORT_SYMBOL(sock_no_getname);
2676 
2677 int sock_no_ioctl(struct socket *sock, unsigned int cmd, unsigned long arg)
2678 {
2679 	return -EOPNOTSUPP;
2680 }
2681 EXPORT_SYMBOL(sock_no_ioctl);
2682 
2683 int sock_no_listen(struct socket *sock, int backlog)
2684 {
2685 	return -EOPNOTSUPP;
2686 }
2687 EXPORT_SYMBOL(sock_no_listen);
2688 
2689 int sock_no_shutdown(struct socket *sock, int how)
2690 {
2691 	return -EOPNOTSUPP;
2692 }
2693 EXPORT_SYMBOL(sock_no_shutdown);
2694 
2695 int sock_no_setsockopt(struct socket *sock, int level, int optname,
2696 		    char __user *optval, unsigned int optlen)
2697 {
2698 	return -EOPNOTSUPP;
2699 }
2700 EXPORT_SYMBOL(sock_no_setsockopt);
2701 
2702 int sock_no_getsockopt(struct socket *sock, int level, int optname,
2703 		    char __user *optval, int __user *optlen)
2704 {
2705 	return -EOPNOTSUPP;
2706 }
2707 EXPORT_SYMBOL(sock_no_getsockopt);
2708 
2709 int sock_no_sendmsg(struct socket *sock, struct msghdr *m, size_t len)
2710 {
2711 	return -EOPNOTSUPP;
2712 }
2713 EXPORT_SYMBOL(sock_no_sendmsg);
2714 
2715 int sock_no_sendmsg_locked(struct sock *sk, struct msghdr *m, size_t len)
2716 {
2717 	return -EOPNOTSUPP;
2718 }
2719 EXPORT_SYMBOL(sock_no_sendmsg_locked);
2720 
2721 int sock_no_recvmsg(struct socket *sock, struct msghdr *m, size_t len,
2722 		    int flags)
2723 {
2724 	return -EOPNOTSUPP;
2725 }
2726 EXPORT_SYMBOL(sock_no_recvmsg);
2727 
2728 int sock_no_mmap(struct file *file, struct socket *sock, struct vm_area_struct *vma)
2729 {
2730 	/* Mirror missing mmap method error code */
2731 	return -ENODEV;
2732 }
2733 EXPORT_SYMBOL(sock_no_mmap);
2734 
2735 ssize_t sock_no_sendpage(struct socket *sock, struct page *page, int offset, size_t size, int flags)
2736 {
2737 	ssize_t res;
2738 	struct msghdr msg = {.msg_flags = flags};
2739 	struct kvec iov;
2740 	char *kaddr = kmap(page);
2741 	iov.iov_base = kaddr + offset;
2742 	iov.iov_len = size;
2743 	res = kernel_sendmsg(sock, &msg, &iov, 1, size);
2744 	kunmap(page);
2745 	return res;
2746 }
2747 EXPORT_SYMBOL(sock_no_sendpage);
2748 
2749 ssize_t sock_no_sendpage_locked(struct sock *sk, struct page *page,
2750 				int offset, size_t size, int flags)
2751 {
2752 	ssize_t res;
2753 	struct msghdr msg = {.msg_flags = flags};
2754 	struct kvec iov;
2755 	char *kaddr = kmap(page);
2756 
2757 	iov.iov_base = kaddr + offset;
2758 	iov.iov_len = size;
2759 	res = kernel_sendmsg_locked(sk, &msg, &iov, 1, size);
2760 	kunmap(page);
2761 	return res;
2762 }
2763 EXPORT_SYMBOL(sock_no_sendpage_locked);
2764 
2765 /*
2766  *	Default Socket Callbacks
2767  */
2768 
2769 static void sock_def_wakeup(struct sock *sk)
2770 {
2771 	struct socket_wq *wq;
2772 
2773 	rcu_read_lock();
2774 	wq = rcu_dereference(sk->sk_wq);
2775 	if (skwq_has_sleeper(wq))
2776 		wake_up_interruptible_all(&wq->wait);
2777 	rcu_read_unlock();
2778 }
2779 
2780 static void sock_def_error_report(struct sock *sk)
2781 {
2782 	struct socket_wq *wq;
2783 
2784 	rcu_read_lock();
2785 	wq = rcu_dereference(sk->sk_wq);
2786 	if (skwq_has_sleeper(wq))
2787 		wake_up_interruptible_poll(&wq->wait, EPOLLERR);
2788 	sk_wake_async(sk, SOCK_WAKE_IO, POLL_ERR);
2789 	rcu_read_unlock();
2790 }
2791 
2792 void sock_def_readable(struct sock *sk)
2793 {
2794 	struct socket_wq *wq;
2795 
2796 	rcu_read_lock();
2797 	wq = rcu_dereference(sk->sk_wq);
2798 	if (skwq_has_sleeper(wq))
2799 		wake_up_interruptible_sync_poll(&wq->wait, EPOLLIN | EPOLLPRI |
2800 						EPOLLRDNORM | EPOLLRDBAND);
2801 	sk_wake_async(sk, SOCK_WAKE_WAITD, POLL_IN);
2802 	rcu_read_unlock();
2803 }
2804 
2805 static void sock_def_write_space(struct sock *sk)
2806 {
2807 	struct socket_wq *wq;
2808 
2809 	rcu_read_lock();
2810 
2811 	/* Do not wake up a writer until he can make "significant"
2812 	 * progress.  --DaveM
2813 	 */
2814 	if ((refcount_read(&sk->sk_wmem_alloc) << 1) <= READ_ONCE(sk->sk_sndbuf)) {
2815 		wq = rcu_dereference(sk->sk_wq);
2816 		if (skwq_has_sleeper(wq))
2817 			wake_up_interruptible_sync_poll(&wq->wait, EPOLLOUT |
2818 						EPOLLWRNORM | EPOLLWRBAND);
2819 
2820 		/* Should agree with poll, otherwise some programs break */
2821 		if (sock_writeable(sk))
2822 			sk_wake_async(sk, SOCK_WAKE_SPACE, POLL_OUT);
2823 	}
2824 
2825 	rcu_read_unlock();
2826 }
2827 
2828 static void sock_def_destruct(struct sock *sk)
2829 {
2830 }
2831 
2832 void sk_send_sigurg(struct sock *sk)
2833 {
2834 	if (sk->sk_socket && sk->sk_socket->file)
2835 		if (send_sigurg(&sk->sk_socket->file->f_owner))
2836 			sk_wake_async(sk, SOCK_WAKE_URG, POLL_PRI);
2837 }
2838 EXPORT_SYMBOL(sk_send_sigurg);
2839 
2840 void sk_reset_timer(struct sock *sk, struct timer_list* timer,
2841 		    unsigned long expires)
2842 {
2843 	if (!mod_timer(timer, expires))
2844 		sock_hold(sk);
2845 }
2846 EXPORT_SYMBOL(sk_reset_timer);
2847 
2848 void sk_stop_timer(struct sock *sk, struct timer_list* timer)
2849 {
2850 	if (del_timer(timer))
2851 		__sock_put(sk);
2852 }
2853 EXPORT_SYMBOL(sk_stop_timer);
2854 
2855 void sock_init_data(struct socket *sock, struct sock *sk)
2856 {
2857 	sk_init_common(sk);
2858 	sk->sk_send_head	=	NULL;
2859 
2860 	timer_setup(&sk->sk_timer, NULL, 0);
2861 
2862 	sk->sk_allocation	=	GFP_KERNEL;
2863 	sk->sk_rcvbuf		=	sysctl_rmem_default;
2864 	sk->sk_sndbuf		=	sysctl_wmem_default;
2865 	sk->sk_state		=	TCP_CLOSE;
2866 	sk_set_socket(sk, sock);
2867 
2868 	sock_set_flag(sk, SOCK_ZAPPED);
2869 
2870 	if (sock) {
2871 		sk->sk_type	=	sock->type;
2872 		RCU_INIT_POINTER(sk->sk_wq, &sock->wq);
2873 		sock->sk	=	sk;
2874 		sk->sk_uid	=	SOCK_INODE(sock)->i_uid;
2875 	} else {
2876 		RCU_INIT_POINTER(sk->sk_wq, NULL);
2877 		sk->sk_uid	=	make_kuid(sock_net(sk)->user_ns, 0);
2878 	}
2879 
2880 	rwlock_init(&sk->sk_callback_lock);
2881 	if (sk->sk_kern_sock)
2882 		lockdep_set_class_and_name(
2883 			&sk->sk_callback_lock,
2884 			af_kern_callback_keys + sk->sk_family,
2885 			af_family_kern_clock_key_strings[sk->sk_family]);
2886 	else
2887 		lockdep_set_class_and_name(
2888 			&sk->sk_callback_lock,
2889 			af_callback_keys + sk->sk_family,
2890 			af_family_clock_key_strings[sk->sk_family]);
2891 
2892 	sk->sk_state_change	=	sock_def_wakeup;
2893 	sk->sk_data_ready	=	sock_def_readable;
2894 	sk->sk_write_space	=	sock_def_write_space;
2895 	sk->sk_error_report	=	sock_def_error_report;
2896 	sk->sk_destruct		=	sock_def_destruct;
2897 
2898 	sk->sk_frag.page	=	NULL;
2899 	sk->sk_frag.offset	=	0;
2900 	sk->sk_peek_off		=	-1;
2901 
2902 	sk->sk_peer_pid 	=	NULL;
2903 	sk->sk_peer_cred	=	NULL;
2904 	sk->sk_write_pending	=	0;
2905 	sk->sk_rcvlowat		=	1;
2906 	sk->sk_rcvtimeo		=	MAX_SCHEDULE_TIMEOUT;
2907 	sk->sk_sndtimeo		=	MAX_SCHEDULE_TIMEOUT;
2908 
2909 	sk->sk_stamp = SK_DEFAULT_STAMP;
2910 #if BITS_PER_LONG==32
2911 	seqlock_init(&sk->sk_stamp_seq);
2912 #endif
2913 	atomic_set(&sk->sk_zckey, 0);
2914 
2915 #ifdef CONFIG_NET_RX_BUSY_POLL
2916 	sk->sk_napi_id		=	0;
2917 	sk->sk_ll_usec		=	sysctl_net_busy_read;
2918 #endif
2919 
2920 	sk->sk_max_pacing_rate = ~0UL;
2921 	sk->sk_pacing_rate = ~0UL;
2922 	WRITE_ONCE(sk->sk_pacing_shift, 10);
2923 	sk->sk_incoming_cpu = -1;
2924 
2925 	sk_rx_queue_clear(sk);
2926 	/*
2927 	 * Before updating sk_refcnt, we must commit prior changes to memory
2928 	 * (Documentation/RCU/rculist_nulls.txt for details)
2929 	 */
2930 	smp_wmb();
2931 	refcount_set(&sk->sk_refcnt, 1);
2932 	atomic_set(&sk->sk_drops, 0);
2933 }
2934 EXPORT_SYMBOL(sock_init_data);
2935 
2936 void lock_sock_nested(struct sock *sk, int subclass)
2937 {
2938 	might_sleep();
2939 	spin_lock_bh(&sk->sk_lock.slock);
2940 	if (sk->sk_lock.owned)
2941 		__lock_sock(sk);
2942 	sk->sk_lock.owned = 1;
2943 	spin_unlock(&sk->sk_lock.slock);
2944 	/*
2945 	 * The sk_lock has mutex_lock() semantics here:
2946 	 */
2947 	mutex_acquire(&sk->sk_lock.dep_map, subclass, 0, _RET_IP_);
2948 	local_bh_enable();
2949 }
2950 EXPORT_SYMBOL(lock_sock_nested);
2951 
2952 void release_sock(struct sock *sk)
2953 {
2954 	spin_lock_bh(&sk->sk_lock.slock);
2955 	if (sk->sk_backlog.tail)
2956 		__release_sock(sk);
2957 
2958 	/* Warning : release_cb() might need to release sk ownership,
2959 	 * ie call sock_release_ownership(sk) before us.
2960 	 */
2961 	if (sk->sk_prot->release_cb)
2962 		sk->sk_prot->release_cb(sk);
2963 
2964 	sock_release_ownership(sk);
2965 	if (waitqueue_active(&sk->sk_lock.wq))
2966 		wake_up(&sk->sk_lock.wq);
2967 	spin_unlock_bh(&sk->sk_lock.slock);
2968 }
2969 EXPORT_SYMBOL(release_sock);
2970 
2971 /**
2972  * lock_sock_fast - fast version of lock_sock
2973  * @sk: socket
2974  *
2975  * This version should be used for very small section, where process wont block
2976  * return false if fast path is taken:
2977  *
2978  *   sk_lock.slock locked, owned = 0, BH disabled
2979  *
2980  * return true if slow path is taken:
2981  *
2982  *   sk_lock.slock unlocked, owned = 1, BH enabled
2983  */
2984 bool lock_sock_fast(struct sock *sk)
2985 {
2986 	might_sleep();
2987 	spin_lock_bh(&sk->sk_lock.slock);
2988 
2989 	if (!sk->sk_lock.owned)
2990 		/*
2991 		 * Note : We must disable BH
2992 		 */
2993 		return false;
2994 
2995 	__lock_sock(sk);
2996 	sk->sk_lock.owned = 1;
2997 	spin_unlock(&sk->sk_lock.slock);
2998 	/*
2999 	 * The sk_lock has mutex_lock() semantics here:
3000 	 */
3001 	mutex_acquire(&sk->sk_lock.dep_map, 0, 0, _RET_IP_);
3002 	local_bh_enable();
3003 	return true;
3004 }
3005 EXPORT_SYMBOL(lock_sock_fast);
3006 
3007 int sock_gettstamp(struct socket *sock, void __user *userstamp,
3008 		   bool timeval, bool time32)
3009 {
3010 	struct sock *sk = sock->sk;
3011 	struct timespec64 ts;
3012 
3013 	sock_enable_timestamp(sk, SOCK_TIMESTAMP);
3014 	ts = ktime_to_timespec64(sock_read_timestamp(sk));
3015 	if (ts.tv_sec == -1)
3016 		return -ENOENT;
3017 	if (ts.tv_sec == 0) {
3018 		ktime_t kt = ktime_get_real();
3019 		sock_write_timestamp(sk, kt);
3020 		ts = ktime_to_timespec64(kt);
3021 	}
3022 
3023 	if (timeval)
3024 		ts.tv_nsec /= 1000;
3025 
3026 #ifdef CONFIG_COMPAT_32BIT_TIME
3027 	if (time32)
3028 		return put_old_timespec32(&ts, userstamp);
3029 #endif
3030 #ifdef CONFIG_SPARC64
3031 	/* beware of padding in sparc64 timeval */
3032 	if (timeval && !in_compat_syscall()) {
3033 		struct __kernel_old_timeval __user tv = {
3034 			.tv_sec = ts.tv_sec,
3035 			.tv_usec = ts.tv_nsec,
3036 		};
3037 		if (copy_to_user(userstamp, &tv, sizeof(tv)))
3038 			return -EFAULT;
3039 		return 0;
3040 	}
3041 #endif
3042 	return put_timespec64(&ts, userstamp);
3043 }
3044 EXPORT_SYMBOL(sock_gettstamp);
3045 
3046 void sock_enable_timestamp(struct sock *sk, enum sock_flags flag)
3047 {
3048 	if (!sock_flag(sk, flag)) {
3049 		unsigned long previous_flags = sk->sk_flags;
3050 
3051 		sock_set_flag(sk, flag);
3052 		/*
3053 		 * we just set one of the two flags which require net
3054 		 * time stamping, but time stamping might have been on
3055 		 * already because of the other one
3056 		 */
3057 		if (sock_needs_netstamp(sk) &&
3058 		    !(previous_flags & SK_FLAGS_TIMESTAMP))
3059 			net_enable_timestamp();
3060 	}
3061 }
3062 
3063 int sock_recv_errqueue(struct sock *sk, struct msghdr *msg, int len,
3064 		       int level, int type)
3065 {
3066 	struct sock_exterr_skb *serr;
3067 	struct sk_buff *skb;
3068 	int copied, err;
3069 
3070 	err = -EAGAIN;
3071 	skb = sock_dequeue_err_skb(sk);
3072 	if (skb == NULL)
3073 		goto out;
3074 
3075 	copied = skb->len;
3076 	if (copied > len) {
3077 		msg->msg_flags |= MSG_TRUNC;
3078 		copied = len;
3079 	}
3080 	err = skb_copy_datagram_msg(skb, 0, msg, copied);
3081 	if (err)
3082 		goto out_free_skb;
3083 
3084 	sock_recv_timestamp(msg, sk, skb);
3085 
3086 	serr = SKB_EXT_ERR(skb);
3087 	put_cmsg(msg, level, type, sizeof(serr->ee), &serr->ee);
3088 
3089 	msg->msg_flags |= MSG_ERRQUEUE;
3090 	err = copied;
3091 
3092 out_free_skb:
3093 	kfree_skb(skb);
3094 out:
3095 	return err;
3096 }
3097 EXPORT_SYMBOL(sock_recv_errqueue);
3098 
3099 /*
3100  *	Get a socket option on an socket.
3101  *
3102  *	FIX: POSIX 1003.1g is very ambiguous here. It states that
3103  *	asynchronous errors should be reported by getsockopt. We assume
3104  *	this means if you specify SO_ERROR (otherwise whats the point of it).
3105  */
3106 int sock_common_getsockopt(struct socket *sock, int level, int optname,
3107 			   char __user *optval, int __user *optlen)
3108 {
3109 	struct sock *sk = sock->sk;
3110 
3111 	return sk->sk_prot->getsockopt(sk, level, optname, optval, optlen);
3112 }
3113 EXPORT_SYMBOL(sock_common_getsockopt);
3114 
3115 #ifdef CONFIG_COMPAT
3116 int compat_sock_common_getsockopt(struct socket *sock, int level, int optname,
3117 				  char __user *optval, int __user *optlen)
3118 {
3119 	struct sock *sk = sock->sk;
3120 
3121 	if (sk->sk_prot->compat_getsockopt != NULL)
3122 		return sk->sk_prot->compat_getsockopt(sk, level, optname,
3123 						      optval, optlen);
3124 	return sk->sk_prot->getsockopt(sk, level, optname, optval, optlen);
3125 }
3126 EXPORT_SYMBOL(compat_sock_common_getsockopt);
3127 #endif
3128 
3129 int sock_common_recvmsg(struct socket *sock, struct msghdr *msg, size_t size,
3130 			int flags)
3131 {
3132 	struct sock *sk = sock->sk;
3133 	int addr_len = 0;
3134 	int err;
3135 
3136 	err = sk->sk_prot->recvmsg(sk, msg, size, flags & MSG_DONTWAIT,
3137 				   flags & ~MSG_DONTWAIT, &addr_len);
3138 	if (err >= 0)
3139 		msg->msg_namelen = addr_len;
3140 	return err;
3141 }
3142 EXPORT_SYMBOL(sock_common_recvmsg);
3143 
3144 /*
3145  *	Set socket options on an inet socket.
3146  */
3147 int sock_common_setsockopt(struct socket *sock, int level, int optname,
3148 			   char __user *optval, unsigned int optlen)
3149 {
3150 	struct sock *sk = sock->sk;
3151 
3152 	return sk->sk_prot->setsockopt(sk, level, optname, optval, optlen);
3153 }
3154 EXPORT_SYMBOL(sock_common_setsockopt);
3155 
3156 #ifdef CONFIG_COMPAT
3157 int compat_sock_common_setsockopt(struct socket *sock, int level, int optname,
3158 				  char __user *optval, unsigned int optlen)
3159 {
3160 	struct sock *sk = sock->sk;
3161 
3162 	if (sk->sk_prot->compat_setsockopt != NULL)
3163 		return sk->sk_prot->compat_setsockopt(sk, level, optname,
3164 						      optval, optlen);
3165 	return sk->sk_prot->setsockopt(sk, level, optname, optval, optlen);
3166 }
3167 EXPORT_SYMBOL(compat_sock_common_setsockopt);
3168 #endif
3169 
3170 void sk_common_release(struct sock *sk)
3171 {
3172 	if (sk->sk_prot->destroy)
3173 		sk->sk_prot->destroy(sk);
3174 
3175 	/*
3176 	 * Observation: when sock_common_release is called, processes have
3177 	 * no access to socket. But net still has.
3178 	 * Step one, detach it from networking:
3179 	 *
3180 	 * A. Remove from hash tables.
3181 	 */
3182 
3183 	sk->sk_prot->unhash(sk);
3184 
3185 	/*
3186 	 * In this point socket cannot receive new packets, but it is possible
3187 	 * that some packets are in flight because some CPU runs receiver and
3188 	 * did hash table lookup before we unhashed socket. They will achieve
3189 	 * receive queue and will be purged by socket destructor.
3190 	 *
3191 	 * Also we still have packets pending on receive queue and probably,
3192 	 * our own packets waiting in device queues. sock_destroy will drain
3193 	 * receive queue, but transmitted packets will delay socket destruction
3194 	 * until the last reference will be released.
3195 	 */
3196 
3197 	sock_orphan(sk);
3198 
3199 	xfrm_sk_free_policy(sk);
3200 
3201 	sk_refcnt_debug_release(sk);
3202 
3203 	sock_put(sk);
3204 }
3205 EXPORT_SYMBOL(sk_common_release);
3206 
3207 void sk_get_meminfo(const struct sock *sk, u32 *mem)
3208 {
3209 	memset(mem, 0, sizeof(*mem) * SK_MEMINFO_VARS);
3210 
3211 	mem[SK_MEMINFO_RMEM_ALLOC] = sk_rmem_alloc_get(sk);
3212 	mem[SK_MEMINFO_RCVBUF] = READ_ONCE(sk->sk_rcvbuf);
3213 	mem[SK_MEMINFO_WMEM_ALLOC] = sk_wmem_alloc_get(sk);
3214 	mem[SK_MEMINFO_SNDBUF] = READ_ONCE(sk->sk_sndbuf);
3215 	mem[SK_MEMINFO_FWD_ALLOC] = sk->sk_forward_alloc;
3216 	mem[SK_MEMINFO_WMEM_QUEUED] = READ_ONCE(sk->sk_wmem_queued);
3217 	mem[SK_MEMINFO_OPTMEM] = atomic_read(&sk->sk_omem_alloc);
3218 	mem[SK_MEMINFO_BACKLOG] = READ_ONCE(sk->sk_backlog.len);
3219 	mem[SK_MEMINFO_DROPS] = atomic_read(&sk->sk_drops);
3220 }
3221 
3222 #ifdef CONFIG_PROC_FS
3223 #define PROTO_INUSE_NR	64	/* should be enough for the first time */
3224 struct prot_inuse {
3225 	int val[PROTO_INUSE_NR];
3226 };
3227 
3228 static DECLARE_BITMAP(proto_inuse_idx, PROTO_INUSE_NR);
3229 
3230 void sock_prot_inuse_add(struct net *net, struct proto *prot, int val)
3231 {
3232 	__this_cpu_add(net->core.prot_inuse->val[prot->inuse_idx], val);
3233 }
3234 EXPORT_SYMBOL_GPL(sock_prot_inuse_add);
3235 
3236 int sock_prot_inuse_get(struct net *net, struct proto *prot)
3237 {
3238 	int cpu, idx = prot->inuse_idx;
3239 	int res = 0;
3240 
3241 	for_each_possible_cpu(cpu)
3242 		res += per_cpu_ptr(net->core.prot_inuse, cpu)->val[idx];
3243 
3244 	return res >= 0 ? res : 0;
3245 }
3246 EXPORT_SYMBOL_GPL(sock_prot_inuse_get);
3247 
3248 static void sock_inuse_add(struct net *net, int val)
3249 {
3250 	this_cpu_add(*net->core.sock_inuse, val);
3251 }
3252 
3253 int sock_inuse_get(struct net *net)
3254 {
3255 	int cpu, res = 0;
3256 
3257 	for_each_possible_cpu(cpu)
3258 		res += *per_cpu_ptr(net->core.sock_inuse, cpu);
3259 
3260 	return res;
3261 }
3262 
3263 EXPORT_SYMBOL_GPL(sock_inuse_get);
3264 
3265 static int __net_init sock_inuse_init_net(struct net *net)
3266 {
3267 	net->core.prot_inuse = alloc_percpu(struct prot_inuse);
3268 	if (net->core.prot_inuse == NULL)
3269 		return -ENOMEM;
3270 
3271 	net->core.sock_inuse = alloc_percpu(int);
3272 	if (net->core.sock_inuse == NULL)
3273 		goto out;
3274 
3275 	return 0;
3276 
3277 out:
3278 	free_percpu(net->core.prot_inuse);
3279 	return -ENOMEM;
3280 }
3281 
3282 static void __net_exit sock_inuse_exit_net(struct net *net)
3283 {
3284 	free_percpu(net->core.prot_inuse);
3285 	free_percpu(net->core.sock_inuse);
3286 }
3287 
3288 static struct pernet_operations net_inuse_ops = {
3289 	.init = sock_inuse_init_net,
3290 	.exit = sock_inuse_exit_net,
3291 };
3292 
3293 static __init int net_inuse_init(void)
3294 {
3295 	if (register_pernet_subsys(&net_inuse_ops))
3296 		panic("Cannot initialize net inuse counters");
3297 
3298 	return 0;
3299 }
3300 
3301 core_initcall(net_inuse_init);
3302 
3303 static int assign_proto_idx(struct proto *prot)
3304 {
3305 	prot->inuse_idx = find_first_zero_bit(proto_inuse_idx, PROTO_INUSE_NR);
3306 
3307 	if (unlikely(prot->inuse_idx == PROTO_INUSE_NR - 1)) {
3308 		pr_err("PROTO_INUSE_NR exhausted\n");
3309 		return -ENOSPC;
3310 	}
3311 
3312 	set_bit(prot->inuse_idx, proto_inuse_idx);
3313 	return 0;
3314 }
3315 
3316 static void release_proto_idx(struct proto *prot)
3317 {
3318 	if (prot->inuse_idx != PROTO_INUSE_NR - 1)
3319 		clear_bit(prot->inuse_idx, proto_inuse_idx);
3320 }
3321 #else
3322 static inline int assign_proto_idx(struct proto *prot)
3323 {
3324 	return 0;
3325 }
3326 
3327 static inline void release_proto_idx(struct proto *prot)
3328 {
3329 }
3330 
3331 static void sock_inuse_add(struct net *net, int val)
3332 {
3333 }
3334 #endif
3335 
3336 static void req_prot_cleanup(struct request_sock_ops *rsk_prot)
3337 {
3338 	if (!rsk_prot)
3339 		return;
3340 	kfree(rsk_prot->slab_name);
3341 	rsk_prot->slab_name = NULL;
3342 	kmem_cache_destroy(rsk_prot->slab);
3343 	rsk_prot->slab = NULL;
3344 }
3345 
3346 static int req_prot_init(const struct proto *prot)
3347 {
3348 	struct request_sock_ops *rsk_prot = prot->rsk_prot;
3349 
3350 	if (!rsk_prot)
3351 		return 0;
3352 
3353 	rsk_prot->slab_name = kasprintf(GFP_KERNEL, "request_sock_%s",
3354 					prot->name);
3355 	if (!rsk_prot->slab_name)
3356 		return -ENOMEM;
3357 
3358 	rsk_prot->slab = kmem_cache_create(rsk_prot->slab_name,
3359 					   rsk_prot->obj_size, 0,
3360 					   SLAB_ACCOUNT | prot->slab_flags,
3361 					   NULL);
3362 
3363 	if (!rsk_prot->slab) {
3364 		pr_crit("%s: Can't create request sock SLAB cache!\n",
3365 			prot->name);
3366 		return -ENOMEM;
3367 	}
3368 	return 0;
3369 }
3370 
3371 int proto_register(struct proto *prot, int alloc_slab)
3372 {
3373 	int ret = -ENOBUFS;
3374 
3375 	if (alloc_slab) {
3376 		prot->slab = kmem_cache_create_usercopy(prot->name,
3377 					prot->obj_size, 0,
3378 					SLAB_HWCACHE_ALIGN | SLAB_ACCOUNT |
3379 					prot->slab_flags,
3380 					prot->useroffset, prot->usersize,
3381 					NULL);
3382 
3383 		if (prot->slab == NULL) {
3384 			pr_crit("%s: Can't create sock SLAB cache!\n",
3385 				prot->name);
3386 			goto out;
3387 		}
3388 
3389 		if (req_prot_init(prot))
3390 			goto out_free_request_sock_slab;
3391 
3392 		if (prot->twsk_prot != NULL) {
3393 			prot->twsk_prot->twsk_slab_name = kasprintf(GFP_KERNEL, "tw_sock_%s", prot->name);
3394 
3395 			if (prot->twsk_prot->twsk_slab_name == NULL)
3396 				goto out_free_request_sock_slab;
3397 
3398 			prot->twsk_prot->twsk_slab =
3399 				kmem_cache_create(prot->twsk_prot->twsk_slab_name,
3400 						  prot->twsk_prot->twsk_obj_size,
3401 						  0,
3402 						  SLAB_ACCOUNT |
3403 						  prot->slab_flags,
3404 						  NULL);
3405 			if (prot->twsk_prot->twsk_slab == NULL)
3406 				goto out_free_timewait_sock_slab_name;
3407 		}
3408 	}
3409 
3410 	mutex_lock(&proto_list_mutex);
3411 	ret = assign_proto_idx(prot);
3412 	if (ret) {
3413 		mutex_unlock(&proto_list_mutex);
3414 		goto out_free_timewait_sock_slab_name;
3415 	}
3416 	list_add(&prot->node, &proto_list);
3417 	mutex_unlock(&proto_list_mutex);
3418 	return ret;
3419 
3420 out_free_timewait_sock_slab_name:
3421 	if (alloc_slab && prot->twsk_prot)
3422 		kfree(prot->twsk_prot->twsk_slab_name);
3423 out_free_request_sock_slab:
3424 	if (alloc_slab) {
3425 		req_prot_cleanup(prot->rsk_prot);
3426 
3427 		kmem_cache_destroy(prot->slab);
3428 		prot->slab = NULL;
3429 	}
3430 out:
3431 	return ret;
3432 }
3433 EXPORT_SYMBOL(proto_register);
3434 
3435 void proto_unregister(struct proto *prot)
3436 {
3437 	mutex_lock(&proto_list_mutex);
3438 	release_proto_idx(prot);
3439 	list_del(&prot->node);
3440 	mutex_unlock(&proto_list_mutex);
3441 
3442 	kmem_cache_destroy(prot->slab);
3443 	prot->slab = NULL;
3444 
3445 	req_prot_cleanup(prot->rsk_prot);
3446 
3447 	if (prot->twsk_prot != NULL && prot->twsk_prot->twsk_slab != NULL) {
3448 		kmem_cache_destroy(prot->twsk_prot->twsk_slab);
3449 		kfree(prot->twsk_prot->twsk_slab_name);
3450 		prot->twsk_prot->twsk_slab = NULL;
3451 	}
3452 }
3453 EXPORT_SYMBOL(proto_unregister);
3454 
3455 int sock_load_diag_module(int family, int protocol)
3456 {
3457 	if (!protocol) {
3458 		if (!sock_is_registered(family))
3459 			return -ENOENT;
3460 
3461 		return request_module("net-pf-%d-proto-%d-type-%d", PF_NETLINK,
3462 				      NETLINK_SOCK_DIAG, family);
3463 	}
3464 
3465 #ifdef CONFIG_INET
3466 	if (family == AF_INET &&
3467 	    protocol != IPPROTO_RAW &&
3468 	    !rcu_access_pointer(inet_protos[protocol]))
3469 		return -ENOENT;
3470 #endif
3471 
3472 	return request_module("net-pf-%d-proto-%d-type-%d-%d", PF_NETLINK,
3473 			      NETLINK_SOCK_DIAG, family, protocol);
3474 }
3475 EXPORT_SYMBOL(sock_load_diag_module);
3476 
3477 #ifdef CONFIG_PROC_FS
3478 static void *proto_seq_start(struct seq_file *seq, loff_t *pos)
3479 	__acquires(proto_list_mutex)
3480 {
3481 	mutex_lock(&proto_list_mutex);
3482 	return seq_list_start_head(&proto_list, *pos);
3483 }
3484 
3485 static void *proto_seq_next(struct seq_file *seq, void *v, loff_t *pos)
3486 {
3487 	return seq_list_next(v, &proto_list, pos);
3488 }
3489 
3490 static void proto_seq_stop(struct seq_file *seq, void *v)
3491 	__releases(proto_list_mutex)
3492 {
3493 	mutex_unlock(&proto_list_mutex);
3494 }
3495 
3496 static char proto_method_implemented(const void *method)
3497 {
3498 	return method == NULL ? 'n' : 'y';
3499 }
3500 static long sock_prot_memory_allocated(struct proto *proto)
3501 {
3502 	return proto->memory_allocated != NULL ? proto_memory_allocated(proto) : -1L;
3503 }
3504 
3505 static const char *sock_prot_memory_pressure(struct proto *proto)
3506 {
3507 	return proto->memory_pressure != NULL ?
3508 	proto_memory_pressure(proto) ? "yes" : "no" : "NI";
3509 }
3510 
3511 static void proto_seq_printf(struct seq_file *seq, struct proto *proto)
3512 {
3513 
3514 	seq_printf(seq, "%-9s %4u %6d  %6ld   %-3s %6u   %-3s  %-10s "
3515 			"%2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c\n",
3516 		   proto->name,
3517 		   proto->obj_size,
3518 		   sock_prot_inuse_get(seq_file_net(seq), proto),
3519 		   sock_prot_memory_allocated(proto),
3520 		   sock_prot_memory_pressure(proto),
3521 		   proto->max_header,
3522 		   proto->slab == NULL ? "no" : "yes",
3523 		   module_name(proto->owner),
3524 		   proto_method_implemented(proto->close),
3525 		   proto_method_implemented(proto->connect),
3526 		   proto_method_implemented(proto->disconnect),
3527 		   proto_method_implemented(proto->accept),
3528 		   proto_method_implemented(proto->ioctl),
3529 		   proto_method_implemented(proto->init),
3530 		   proto_method_implemented(proto->destroy),
3531 		   proto_method_implemented(proto->shutdown),
3532 		   proto_method_implemented(proto->setsockopt),
3533 		   proto_method_implemented(proto->getsockopt),
3534 		   proto_method_implemented(proto->sendmsg),
3535 		   proto_method_implemented(proto->recvmsg),
3536 		   proto_method_implemented(proto->sendpage),
3537 		   proto_method_implemented(proto->bind),
3538 		   proto_method_implemented(proto->backlog_rcv),
3539 		   proto_method_implemented(proto->hash),
3540 		   proto_method_implemented(proto->unhash),
3541 		   proto_method_implemented(proto->get_port),
3542 		   proto_method_implemented(proto->enter_memory_pressure));
3543 }
3544 
3545 static int proto_seq_show(struct seq_file *seq, void *v)
3546 {
3547 	if (v == &proto_list)
3548 		seq_printf(seq, "%-9s %-4s %-8s %-6s %-5s %-7s %-4s %-10s %s",
3549 			   "protocol",
3550 			   "size",
3551 			   "sockets",
3552 			   "memory",
3553 			   "press",
3554 			   "maxhdr",
3555 			   "slab",
3556 			   "module",
3557 			   "cl co di ac io in de sh ss gs se re sp bi br ha uh gp em\n");
3558 	else
3559 		proto_seq_printf(seq, list_entry(v, struct proto, node));
3560 	return 0;
3561 }
3562 
3563 static const struct seq_operations proto_seq_ops = {
3564 	.start  = proto_seq_start,
3565 	.next   = proto_seq_next,
3566 	.stop   = proto_seq_stop,
3567 	.show   = proto_seq_show,
3568 };
3569 
3570 static __net_init int proto_init_net(struct net *net)
3571 {
3572 	if (!proc_create_net("protocols", 0444, net->proc_net, &proto_seq_ops,
3573 			sizeof(struct seq_net_private)))
3574 		return -ENOMEM;
3575 
3576 	return 0;
3577 }
3578 
3579 static __net_exit void proto_exit_net(struct net *net)
3580 {
3581 	remove_proc_entry("protocols", net->proc_net);
3582 }
3583 
3584 
3585 static __net_initdata struct pernet_operations proto_net_ops = {
3586 	.init = proto_init_net,
3587 	.exit = proto_exit_net,
3588 };
3589 
3590 static int __init proto_init(void)
3591 {
3592 	return register_pernet_subsys(&proto_net_ops);
3593 }
3594 
3595 subsys_initcall(proto_init);
3596 
3597 #endif /* PROC_FS */
3598 
3599 #ifdef CONFIG_NET_RX_BUSY_POLL
3600 bool sk_busy_loop_end(void *p, unsigned long start_time)
3601 {
3602 	struct sock *sk = p;
3603 
3604 	return !skb_queue_empty_lockless(&sk->sk_receive_queue) ||
3605 	       sk_busy_loop_timeout(sk, start_time);
3606 }
3607 EXPORT_SYMBOL(sk_busy_loop_end);
3608 #endif /* CONFIG_NET_RX_BUSY_POLL */
3609