xref: /openbmc/linux/net/ipv4/tcp.c (revision ee8a99bd)
1 /*
2  * INET		An implementation of the TCP/IP protocol suite for the LINUX
3  *		operating system.  INET is implemented using the  BSD Socket
4  *		interface as the means of communication with the user level.
5  *
6  *		Implementation of the Transmission Control Protocol(TCP).
7  *
8  * Authors:	Ross Biro
9  *		Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
10  *		Mark Evans, <evansmp@uhura.aston.ac.uk>
11  *		Corey Minyard <wf-rch!minyard@relay.EU.net>
12  *		Florian La Roche, <flla@stud.uni-sb.de>
13  *		Charles Hedrick, <hedrick@klinzhai.rutgers.edu>
14  *		Linus Torvalds, <torvalds@cs.helsinki.fi>
15  *		Alan Cox, <gw4pts@gw4pts.ampr.org>
16  *		Matthew Dillon, <dillon@apollo.west.oic.com>
17  *		Arnt Gulbrandsen, <agulbra@nvg.unit.no>
18  *		Jorge Cwik, <jorge@laser.satlink.net>
19  *
20  * Fixes:
21  *		Alan Cox	:	Numerous verify_area() calls
22  *		Alan Cox	:	Set the ACK bit on a reset
23  *		Alan Cox	:	Stopped it crashing if it closed while
24  *					sk->inuse=1 and was trying to connect
25  *					(tcp_err()).
26  *		Alan Cox	:	All icmp error handling was broken
27  *					pointers passed where wrong and the
28  *					socket was looked up backwards. Nobody
29  *					tested any icmp error code obviously.
30  *		Alan Cox	:	tcp_err() now handled properly. It
31  *					wakes people on errors. poll
32  *					behaves and the icmp error race
33  *					has gone by moving it into sock.c
34  *		Alan Cox	:	tcp_send_reset() fixed to work for
35  *					everything not just packets for
36  *					unknown sockets.
37  *		Alan Cox	:	tcp option processing.
38  *		Alan Cox	:	Reset tweaked (still not 100%) [Had
39  *					syn rule wrong]
40  *		Herp Rosmanith  :	More reset fixes
41  *		Alan Cox	:	No longer acks invalid rst frames.
42  *					Acking any kind of RST is right out.
43  *		Alan Cox	:	Sets an ignore me flag on an rst
44  *					receive otherwise odd bits of prattle
45  *					escape still
46  *		Alan Cox	:	Fixed another acking RST frame bug.
47  *					Should stop LAN workplace lockups.
48  *		Alan Cox	: 	Some tidyups using the new skb list
49  *					facilities
50  *		Alan Cox	:	sk->keepopen now seems to work
51  *		Alan Cox	:	Pulls options out correctly on accepts
52  *		Alan Cox	:	Fixed assorted sk->rqueue->next errors
53  *		Alan Cox	:	PSH doesn't end a TCP read. Switched a
54  *					bit to skb ops.
55  *		Alan Cox	:	Tidied tcp_data to avoid a potential
56  *					nasty.
57  *		Alan Cox	:	Added some better commenting, as the
58  *					tcp is hard to follow
59  *		Alan Cox	:	Removed incorrect check for 20 * psh
60  *	Michael O'Reilly	:	ack < copied bug fix.
61  *	Johannes Stille		:	Misc tcp fixes (not all in yet).
62  *		Alan Cox	:	FIN with no memory -> CRASH
63  *		Alan Cox	:	Added socket option proto entries.
64  *					Also added awareness of them to accept.
65  *		Alan Cox	:	Added TCP options (SOL_TCP)
66  *		Alan Cox	:	Switched wakeup calls to callbacks,
67  *					so the kernel can layer network
68  *					sockets.
69  *		Alan Cox	:	Use ip_tos/ip_ttl settings.
70  *		Alan Cox	:	Handle FIN (more) properly (we hope).
71  *		Alan Cox	:	RST frames sent on unsynchronised
72  *					state ack error.
73  *		Alan Cox	:	Put in missing check for SYN bit.
74  *		Alan Cox	:	Added tcp_select_window() aka NET2E
75  *					window non shrink trick.
76  *		Alan Cox	:	Added a couple of small NET2E timer
77  *					fixes
78  *		Charles Hedrick :	TCP fixes
79  *		Toomas Tamm	:	TCP window fixes
80  *		Alan Cox	:	Small URG fix to rlogin ^C ack fight
81  *		Charles Hedrick	:	Rewrote most of it to actually work
82  *		Linus		:	Rewrote tcp_read() and URG handling
83  *					completely
84  *		Gerhard Koerting:	Fixed some missing timer handling
85  *		Matthew Dillon  :	Reworked TCP machine states as per RFC
86  *		Gerhard Koerting:	PC/TCP workarounds
87  *		Adam Caldwell	:	Assorted timer/timing errors
88  *		Matthew Dillon	:	Fixed another RST bug
89  *		Alan Cox	:	Move to kernel side addressing changes.
90  *		Alan Cox	:	Beginning work on TCP fastpathing
91  *					(not yet usable)
92  *		Arnt Gulbrandsen:	Turbocharged tcp_check() routine.
93  *		Alan Cox	:	TCP fast path debugging
94  *		Alan Cox	:	Window clamping
95  *		Michael Riepe	:	Bug in tcp_check()
96  *		Matt Dillon	:	More TCP improvements and RST bug fixes
97  *		Matt Dillon	:	Yet more small nasties remove from the
98  *					TCP code (Be very nice to this man if
99  *					tcp finally works 100%) 8)
100  *		Alan Cox	:	BSD accept semantics.
101  *		Alan Cox	:	Reset on closedown bug.
102  *	Peter De Schrijver	:	ENOTCONN check missing in tcp_sendto().
103  *		Michael Pall	:	Handle poll() after URG properly in
104  *					all cases.
105  *		Michael Pall	:	Undo the last fix in tcp_read_urg()
106  *					(multi URG PUSH broke rlogin).
107  *		Michael Pall	:	Fix the multi URG PUSH problem in
108  *					tcp_readable(), poll() after URG
109  *					works now.
110  *		Michael Pall	:	recv(...,MSG_OOB) never blocks in the
111  *					BSD api.
112  *		Alan Cox	:	Changed the semantics of sk->socket to
113  *					fix a race and a signal problem with
114  *					accept() and async I/O.
115  *		Alan Cox	:	Relaxed the rules on tcp_sendto().
116  *		Yury Shevchuk	:	Really fixed accept() blocking problem.
117  *		Craig I. Hagan  :	Allow for BSD compatible TIME_WAIT for
118  *					clients/servers which listen in on
119  *					fixed ports.
120  *		Alan Cox	:	Cleaned the above up and shrank it to
121  *					a sensible code size.
122  *		Alan Cox	:	Self connect lockup fix.
123  *		Alan Cox	:	No connect to multicast.
124  *		Ross Biro	:	Close unaccepted children on master
125  *					socket close.
126  *		Alan Cox	:	Reset tracing code.
127  *		Alan Cox	:	Spurious resets on shutdown.
128  *		Alan Cox	:	Giant 15 minute/60 second timer error
129  *		Alan Cox	:	Small whoops in polling before an
130  *					accept.
131  *		Alan Cox	:	Kept the state trace facility since
132  *					it's handy for debugging.
133  *		Alan Cox	:	More reset handler fixes.
134  *		Alan Cox	:	Started rewriting the code based on
135  *					the RFC's for other useful protocol
136  *					references see: Comer, KA9Q NOS, and
137  *					for a reference on the difference
138  *					between specifications and how BSD
139  *					works see the 4.4lite source.
140  *		A.N.Kuznetsov	:	Don't time wait on completion of tidy
141  *					close.
142  *		Linus Torvalds	:	Fin/Shutdown & copied_seq changes.
143  *		Linus Torvalds	:	Fixed BSD port reuse to work first syn
144  *		Alan Cox	:	Reimplemented timers as per the RFC
145  *					and using multiple timers for sanity.
146  *		Alan Cox	:	Small bug fixes, and a lot of new
147  *					comments.
148  *		Alan Cox	:	Fixed dual reader crash by locking
149  *					the buffers (much like datagram.c)
150  *		Alan Cox	:	Fixed stuck sockets in probe. A probe
151  *					now gets fed up of retrying without
152  *					(even a no space) answer.
153  *		Alan Cox	:	Extracted closing code better
154  *		Alan Cox	:	Fixed the closing state machine to
155  *					resemble the RFC.
156  *		Alan Cox	:	More 'per spec' fixes.
157  *		Jorge Cwik	:	Even faster checksumming.
158  *		Alan Cox	:	tcp_data() doesn't ack illegal PSH
159  *					only frames. At least one pc tcp stack
160  *					generates them.
161  *		Alan Cox	:	Cache last socket.
162  *		Alan Cox	:	Per route irtt.
163  *		Matt Day	:	poll()->select() match BSD precisely on error
164  *		Alan Cox	:	New buffers
165  *		Marc Tamsky	:	Various sk->prot->retransmits and
166  *					sk->retransmits misupdating fixed.
167  *					Fixed tcp_write_timeout: stuck close,
168  *					and TCP syn retries gets used now.
169  *		Mark Yarvis	:	In tcp_read_wakeup(), don't send an
170  *					ack if state is TCP_CLOSED.
171  *		Alan Cox	:	Look up device on a retransmit - routes may
172  *					change. Doesn't yet cope with MSS shrink right
173  *					but it's a start!
174  *		Marc Tamsky	:	Closing in closing fixes.
175  *		Mike Shaver	:	RFC1122 verifications.
176  *		Alan Cox	:	rcv_saddr errors.
177  *		Alan Cox	:	Block double connect().
178  *		Alan Cox	:	Small hooks for enSKIP.
179  *		Alexey Kuznetsov:	Path MTU discovery.
180  *		Alan Cox	:	Support soft errors.
181  *		Alan Cox	:	Fix MTU discovery pathological case
182  *					when the remote claims no mtu!
183  *		Marc Tamsky	:	TCP_CLOSE fix.
184  *		Colin (G3TNE)	:	Send a reset on syn ack replies in
185  *					window but wrong (fixes NT lpd problems)
186  *		Pedro Roque	:	Better TCP window handling, delayed ack.
187  *		Joerg Reuter	:	No modification of locked buffers in
188  *					tcp_do_retransmit()
189  *		Eric Schenk	:	Changed receiver side silly window
190  *					avoidance algorithm to BSD style
191  *					algorithm. This doubles throughput
192  *					against machines running Solaris,
193  *					and seems to result in general
194  *					improvement.
195  *	Stefan Magdalinski	:	adjusted tcp_readable() to fix FIONREAD
196  *	Willy Konynenberg	:	Transparent proxying support.
197  *	Mike McLagan		:	Routing by source
198  *		Keith Owens	:	Do proper merging with partial SKB's in
199  *					tcp_do_sendmsg to avoid burstiness.
200  *		Eric Schenk	:	Fix fast close down bug with
201  *					shutdown() followed by close().
202  *		Andi Kleen 	:	Make poll agree with SIGIO
203  *	Salvatore Sanfilippo	:	Support SO_LINGER with linger == 1 and
204  *					lingertime == 0 (RFC 793 ABORT Call)
205  *	Hirokazu Takahashi	:	Use copy_from_user() instead of
206  *					csum_and_copy_from_user() if possible.
207  *
208  *		This program is free software; you can redistribute it and/or
209  *		modify it under the terms of the GNU General Public License
210  *		as published by the Free Software Foundation; either version
211  *		2 of the License, or(at your option) any later version.
212  *
213  * Description of States:
214  *
215  *	TCP_SYN_SENT		sent a connection request, waiting for ack
216  *
217  *	TCP_SYN_RECV		received a connection request, sent ack,
218  *				waiting for final ack in three-way handshake.
219  *
220  *	TCP_ESTABLISHED		connection established
221  *
222  *	TCP_FIN_WAIT1		our side has shutdown, waiting to complete
223  *				transmission of remaining buffered data
224  *
225  *	TCP_FIN_WAIT2		all buffered data sent, waiting for remote
226  *				to shutdown
227  *
228  *	TCP_CLOSING		both sides have shutdown but we still have
229  *				data we have to finish sending
230  *
231  *	TCP_TIME_WAIT		timeout to catch resent junk before entering
232  *				closed, can only be entered from FIN_WAIT2
233  *				or CLOSING.  Required because the other end
234  *				may not have gotten our last ACK causing it
235  *				to retransmit the data packet (which we ignore)
236  *
237  *	TCP_CLOSE_WAIT		remote side has shutdown and is waiting for
238  *				us to finish writing our data and to shutdown
239  *				(we have to close() to move on to LAST_ACK)
240  *
241  *	TCP_LAST_ACK		out side has shutdown after remote has
242  *				shutdown.  There may still be data in our
243  *				buffer that we have to finish sending
244  *
245  *	TCP_CLOSE		socket is finished
246  */
247 
248 #define pr_fmt(fmt) "TCP: " fmt
249 
250 #include <linux/kernel.h>
251 #include <linux/module.h>
252 #include <linux/types.h>
253 #include <linux/fcntl.h>
254 #include <linux/poll.h>
255 #include <linux/init.h>
256 #include <linux/fs.h>
257 #include <linux/skbuff.h>
258 #include <linux/scatterlist.h>
259 #include <linux/splice.h>
260 #include <linux/net.h>
261 #include <linux/socket.h>
262 #include <linux/random.h>
263 #include <linux/bootmem.h>
264 #include <linux/highmem.h>
265 #include <linux/swap.h>
266 #include <linux/cache.h>
267 #include <linux/err.h>
268 #include <linux/crypto.h>
269 #include <linux/time.h>
270 #include <linux/slab.h>
271 
272 #include <net/icmp.h>
273 #include <net/inet_common.h>
274 #include <net/tcp.h>
275 #include <net/xfrm.h>
276 #include <net/ip.h>
277 #include <net/netdma.h>
278 #include <net/sock.h>
279 
280 #include <asm/uaccess.h>
281 #include <asm/ioctls.h>
282 #include <net/busy_poll.h>
283 
284 int sysctl_tcp_fin_timeout __read_mostly = TCP_FIN_TIMEOUT;
285 
286 struct percpu_counter tcp_orphan_count;
287 EXPORT_SYMBOL_GPL(tcp_orphan_count);
288 
289 int sysctl_tcp_wmem[3] __read_mostly;
290 int sysctl_tcp_rmem[3] __read_mostly;
291 
292 EXPORT_SYMBOL(sysctl_tcp_rmem);
293 EXPORT_SYMBOL(sysctl_tcp_wmem);
294 
295 atomic_long_t tcp_memory_allocated;	/* Current allocated memory. */
296 EXPORT_SYMBOL(tcp_memory_allocated);
297 
298 /*
299  * Current number of TCP sockets.
300  */
301 struct percpu_counter tcp_sockets_allocated;
302 EXPORT_SYMBOL(tcp_sockets_allocated);
303 
304 /*
305  * TCP splice context
306  */
307 struct tcp_splice_state {
308 	struct pipe_inode_info *pipe;
309 	size_t len;
310 	unsigned int flags;
311 };
312 
313 /*
314  * Pressure flag: try to collapse.
315  * Technical note: it is used by multiple contexts non atomically.
316  * All the __sk_mem_schedule() is of this nature: accounting
317  * is strict, actions are advisory and have some latency.
318  */
319 int tcp_memory_pressure __read_mostly;
320 EXPORT_SYMBOL(tcp_memory_pressure);
321 
322 void tcp_enter_memory_pressure(struct sock *sk)
323 {
324 	if (!tcp_memory_pressure) {
325 		NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPMEMORYPRESSURES);
326 		tcp_memory_pressure = 1;
327 	}
328 }
329 EXPORT_SYMBOL(tcp_enter_memory_pressure);
330 
331 /* Convert seconds to retransmits based on initial and max timeout */
332 static u8 secs_to_retrans(int seconds, int timeout, int rto_max)
333 {
334 	u8 res = 0;
335 
336 	if (seconds > 0) {
337 		int period = timeout;
338 
339 		res = 1;
340 		while (seconds > period && res < 255) {
341 			res++;
342 			timeout <<= 1;
343 			if (timeout > rto_max)
344 				timeout = rto_max;
345 			period += timeout;
346 		}
347 	}
348 	return res;
349 }
350 
351 /* Convert retransmits to seconds based on initial and max timeout */
352 static int retrans_to_secs(u8 retrans, int timeout, int rto_max)
353 {
354 	int period = 0;
355 
356 	if (retrans > 0) {
357 		period = timeout;
358 		while (--retrans) {
359 			timeout <<= 1;
360 			if (timeout > rto_max)
361 				timeout = rto_max;
362 			period += timeout;
363 		}
364 	}
365 	return period;
366 }
367 
368 /* Address-family independent initialization for a tcp_sock.
369  *
370  * NOTE: A lot of things set to zero explicitly by call to
371  *       sk_alloc() so need not be done here.
372  */
373 void tcp_init_sock(struct sock *sk)
374 {
375 	struct inet_connection_sock *icsk = inet_csk(sk);
376 	struct tcp_sock *tp = tcp_sk(sk);
377 
378 	skb_queue_head_init(&tp->out_of_order_queue);
379 	tcp_init_xmit_timers(sk);
380 	tcp_prequeue_init(tp);
381 	INIT_LIST_HEAD(&tp->tsq_node);
382 
383 	icsk->icsk_rto = TCP_TIMEOUT_INIT;
384 	tp->mdev = TCP_TIMEOUT_INIT;
385 
386 	/* So many TCP implementations out there (incorrectly) count the
387 	 * initial SYN frame in their delayed-ACK and congestion control
388 	 * algorithms that we must have the following bandaid to talk
389 	 * efficiently to them.  -DaveM
390 	 */
391 	tp->snd_cwnd = TCP_INIT_CWND;
392 
393 	/* See draft-stevens-tcpca-spec-01 for discussion of the
394 	 * initialization of these values.
395 	 */
396 	tp->snd_ssthresh = TCP_INFINITE_SSTHRESH;
397 	tp->snd_cwnd_clamp = ~0;
398 	tp->mss_cache = TCP_MSS_DEFAULT;
399 
400 	tp->reordering = sysctl_tcp_reordering;
401 	tcp_enable_early_retrans(tp);
402 	icsk->icsk_ca_ops = &tcp_init_congestion_ops;
403 
404 	tp->tsoffset = 0;
405 
406 	sk->sk_state = TCP_CLOSE;
407 
408 	sk->sk_write_space = sk_stream_write_space;
409 	sock_set_flag(sk, SOCK_USE_WRITE_QUEUE);
410 
411 	icsk->icsk_sync_mss = tcp_sync_mss;
412 
413 	/* Presumed zeroed, in order of appearance:
414 	 *	cookie_in_always, cookie_out_never,
415 	 *	s_data_constant, s_data_in, s_data_out
416 	 */
417 	sk->sk_sndbuf = sysctl_tcp_wmem[1];
418 	sk->sk_rcvbuf = sysctl_tcp_rmem[1];
419 
420 	local_bh_disable();
421 	sock_update_memcg(sk);
422 	sk_sockets_allocated_inc(sk);
423 	local_bh_enable();
424 }
425 EXPORT_SYMBOL(tcp_init_sock);
426 
427 /*
428  *	Wait for a TCP event.
429  *
430  *	Note that we don't need to lock the socket, as the upper poll layers
431  *	take care of normal races (between the test and the event) and we don't
432  *	go look at any of the socket buffers directly.
433  */
434 unsigned int tcp_poll(struct file *file, struct socket *sock, poll_table *wait)
435 {
436 	unsigned int mask;
437 	struct sock *sk = sock->sk;
438 	const struct tcp_sock *tp = tcp_sk(sk);
439 
440 	sock_rps_record_flow(sk);
441 
442 	sock_poll_wait(file, sk_sleep(sk), wait);
443 	if (sk->sk_state == TCP_LISTEN)
444 		return inet_csk_listen_poll(sk);
445 
446 	/* Socket is not locked. We are protected from async events
447 	 * by poll logic and correct handling of state changes
448 	 * made by other threads is impossible in any case.
449 	 */
450 
451 	mask = 0;
452 
453 	/*
454 	 * POLLHUP is certainly not done right. But poll() doesn't
455 	 * have a notion of HUP in just one direction, and for a
456 	 * socket the read side is more interesting.
457 	 *
458 	 * Some poll() documentation says that POLLHUP is incompatible
459 	 * with the POLLOUT/POLLWR flags, so somebody should check this
460 	 * all. But careful, it tends to be safer to return too many
461 	 * bits than too few, and you can easily break real applications
462 	 * if you don't tell them that something has hung up!
463 	 *
464 	 * Check-me.
465 	 *
466 	 * Check number 1. POLLHUP is _UNMASKABLE_ event (see UNIX98 and
467 	 * our fs/select.c). It means that after we received EOF,
468 	 * poll always returns immediately, making impossible poll() on write()
469 	 * in state CLOSE_WAIT. One solution is evident --- to set POLLHUP
470 	 * if and only if shutdown has been made in both directions.
471 	 * Actually, it is interesting to look how Solaris and DUX
472 	 * solve this dilemma. I would prefer, if POLLHUP were maskable,
473 	 * then we could set it on SND_SHUTDOWN. BTW examples given
474 	 * in Stevens' books assume exactly this behaviour, it explains
475 	 * why POLLHUP is incompatible with POLLOUT.	--ANK
476 	 *
477 	 * NOTE. Check for TCP_CLOSE is added. The goal is to prevent
478 	 * blocking on fresh not-connected or disconnected socket. --ANK
479 	 */
480 	if (sk->sk_shutdown == SHUTDOWN_MASK || sk->sk_state == TCP_CLOSE)
481 		mask |= POLLHUP;
482 	if (sk->sk_shutdown & RCV_SHUTDOWN)
483 		mask |= POLLIN | POLLRDNORM | POLLRDHUP;
484 
485 	/* Connected or passive Fast Open socket? */
486 	if (sk->sk_state != TCP_SYN_SENT &&
487 	    (sk->sk_state != TCP_SYN_RECV || tp->fastopen_rsk != NULL)) {
488 		int target = sock_rcvlowat(sk, 0, INT_MAX);
489 
490 		if (tp->urg_seq == tp->copied_seq &&
491 		    !sock_flag(sk, SOCK_URGINLINE) &&
492 		    tp->urg_data)
493 			target++;
494 
495 		/* Potential race condition. If read of tp below will
496 		 * escape above sk->sk_state, we can be illegally awaken
497 		 * in SYN_* states. */
498 		if (tp->rcv_nxt - tp->copied_seq >= target)
499 			mask |= POLLIN | POLLRDNORM;
500 
501 		if (!(sk->sk_shutdown & SEND_SHUTDOWN)) {
502 			if (sk_stream_wspace(sk) >= sk_stream_min_wspace(sk)) {
503 				mask |= POLLOUT | POLLWRNORM;
504 			} else {  /* send SIGIO later */
505 				set_bit(SOCK_ASYNC_NOSPACE,
506 					&sk->sk_socket->flags);
507 				set_bit(SOCK_NOSPACE, &sk->sk_socket->flags);
508 
509 				/* Race breaker. If space is freed after
510 				 * wspace test but before the flags are set,
511 				 * IO signal will be lost.
512 				 */
513 				if (sk_stream_wspace(sk) >= sk_stream_min_wspace(sk))
514 					mask |= POLLOUT | POLLWRNORM;
515 			}
516 		} else
517 			mask |= POLLOUT | POLLWRNORM;
518 
519 		if (tp->urg_data & TCP_URG_VALID)
520 			mask |= POLLPRI;
521 	}
522 	/* This barrier is coupled with smp_wmb() in tcp_reset() */
523 	smp_rmb();
524 	if (sk->sk_err)
525 		mask |= POLLERR;
526 
527 	return mask;
528 }
529 EXPORT_SYMBOL(tcp_poll);
530 
531 int tcp_ioctl(struct sock *sk, int cmd, unsigned long arg)
532 {
533 	struct tcp_sock *tp = tcp_sk(sk);
534 	int answ;
535 	bool slow;
536 
537 	switch (cmd) {
538 	case SIOCINQ:
539 		if (sk->sk_state == TCP_LISTEN)
540 			return -EINVAL;
541 
542 		slow = lock_sock_fast(sk);
543 		if ((1 << sk->sk_state) & (TCPF_SYN_SENT | TCPF_SYN_RECV))
544 			answ = 0;
545 		else if (sock_flag(sk, SOCK_URGINLINE) ||
546 			 !tp->urg_data ||
547 			 before(tp->urg_seq, tp->copied_seq) ||
548 			 !before(tp->urg_seq, tp->rcv_nxt)) {
549 
550 			answ = tp->rcv_nxt - tp->copied_seq;
551 
552 			/* Subtract 1, if FIN was received */
553 			if (answ && sock_flag(sk, SOCK_DONE))
554 				answ--;
555 		} else
556 			answ = tp->urg_seq - tp->copied_seq;
557 		unlock_sock_fast(sk, slow);
558 		break;
559 	case SIOCATMARK:
560 		answ = tp->urg_data && tp->urg_seq == tp->copied_seq;
561 		break;
562 	case SIOCOUTQ:
563 		if (sk->sk_state == TCP_LISTEN)
564 			return -EINVAL;
565 
566 		if ((1 << sk->sk_state) & (TCPF_SYN_SENT | TCPF_SYN_RECV))
567 			answ = 0;
568 		else
569 			answ = tp->write_seq - tp->snd_una;
570 		break;
571 	case SIOCOUTQNSD:
572 		if (sk->sk_state == TCP_LISTEN)
573 			return -EINVAL;
574 
575 		if ((1 << sk->sk_state) & (TCPF_SYN_SENT | TCPF_SYN_RECV))
576 			answ = 0;
577 		else
578 			answ = tp->write_seq - tp->snd_nxt;
579 		break;
580 	default:
581 		return -ENOIOCTLCMD;
582 	}
583 
584 	return put_user(answ, (int __user *)arg);
585 }
586 EXPORT_SYMBOL(tcp_ioctl);
587 
588 static inline void tcp_mark_push(struct tcp_sock *tp, struct sk_buff *skb)
589 {
590 	TCP_SKB_CB(skb)->tcp_flags |= TCPHDR_PSH;
591 	tp->pushed_seq = tp->write_seq;
592 }
593 
594 static inline bool forced_push(const struct tcp_sock *tp)
595 {
596 	return after(tp->write_seq, tp->pushed_seq + (tp->max_window >> 1));
597 }
598 
599 static inline void skb_entail(struct sock *sk, struct sk_buff *skb)
600 {
601 	struct tcp_sock *tp = tcp_sk(sk);
602 	struct tcp_skb_cb *tcb = TCP_SKB_CB(skb);
603 
604 	skb->csum    = 0;
605 	tcb->seq     = tcb->end_seq = tp->write_seq;
606 	tcb->tcp_flags = TCPHDR_ACK;
607 	tcb->sacked  = 0;
608 	skb_header_release(skb);
609 	tcp_add_write_queue_tail(sk, skb);
610 	sk->sk_wmem_queued += skb->truesize;
611 	sk_mem_charge(sk, skb->truesize);
612 	if (tp->nonagle & TCP_NAGLE_PUSH)
613 		tp->nonagle &= ~TCP_NAGLE_PUSH;
614 }
615 
616 static inline void tcp_mark_urg(struct tcp_sock *tp, int flags)
617 {
618 	if (flags & MSG_OOB)
619 		tp->snd_up = tp->write_seq;
620 }
621 
622 static inline void tcp_push(struct sock *sk, int flags, int mss_now,
623 			    int nonagle)
624 {
625 	if (tcp_send_head(sk)) {
626 		struct tcp_sock *tp = tcp_sk(sk);
627 
628 		if (!(flags & MSG_MORE) || forced_push(tp))
629 			tcp_mark_push(tp, tcp_write_queue_tail(sk));
630 
631 		tcp_mark_urg(tp, flags);
632 		__tcp_push_pending_frames(sk, mss_now,
633 					  (flags & MSG_MORE) ? TCP_NAGLE_CORK : nonagle);
634 	}
635 }
636 
637 static int tcp_splice_data_recv(read_descriptor_t *rd_desc, struct sk_buff *skb,
638 				unsigned int offset, size_t len)
639 {
640 	struct tcp_splice_state *tss = rd_desc->arg.data;
641 	int ret;
642 
643 	ret = skb_splice_bits(skb, offset, tss->pipe, min(rd_desc->count, len),
644 			      tss->flags);
645 	if (ret > 0)
646 		rd_desc->count -= ret;
647 	return ret;
648 }
649 
650 static int __tcp_splice_read(struct sock *sk, struct tcp_splice_state *tss)
651 {
652 	/* Store TCP splice context information in read_descriptor_t. */
653 	read_descriptor_t rd_desc = {
654 		.arg.data = tss,
655 		.count	  = tss->len,
656 	};
657 
658 	return tcp_read_sock(sk, &rd_desc, tcp_splice_data_recv);
659 }
660 
661 /**
662  *  tcp_splice_read - splice data from TCP socket to a pipe
663  * @sock:	socket to splice from
664  * @ppos:	position (not valid)
665  * @pipe:	pipe to splice to
666  * @len:	number of bytes to splice
667  * @flags:	splice modifier flags
668  *
669  * Description:
670  *    Will read pages from given socket and fill them into a pipe.
671  *
672  **/
673 ssize_t tcp_splice_read(struct socket *sock, loff_t *ppos,
674 			struct pipe_inode_info *pipe, size_t len,
675 			unsigned int flags)
676 {
677 	struct sock *sk = sock->sk;
678 	struct tcp_splice_state tss = {
679 		.pipe = pipe,
680 		.len = len,
681 		.flags = flags,
682 	};
683 	long timeo;
684 	ssize_t spliced;
685 	int ret;
686 
687 	sock_rps_record_flow(sk);
688 	/*
689 	 * We can't seek on a socket input
690 	 */
691 	if (unlikely(*ppos))
692 		return -ESPIPE;
693 
694 	ret = spliced = 0;
695 
696 	lock_sock(sk);
697 
698 	timeo = sock_rcvtimeo(sk, sock->file->f_flags & O_NONBLOCK);
699 	while (tss.len) {
700 		ret = __tcp_splice_read(sk, &tss);
701 		if (ret < 0)
702 			break;
703 		else if (!ret) {
704 			if (spliced)
705 				break;
706 			if (sock_flag(sk, SOCK_DONE))
707 				break;
708 			if (sk->sk_err) {
709 				ret = sock_error(sk);
710 				break;
711 			}
712 			if (sk->sk_shutdown & RCV_SHUTDOWN)
713 				break;
714 			if (sk->sk_state == TCP_CLOSE) {
715 				/*
716 				 * This occurs when user tries to read
717 				 * from never connected socket.
718 				 */
719 				if (!sock_flag(sk, SOCK_DONE))
720 					ret = -ENOTCONN;
721 				break;
722 			}
723 			if (!timeo) {
724 				ret = -EAGAIN;
725 				break;
726 			}
727 			sk_wait_data(sk, &timeo);
728 			if (signal_pending(current)) {
729 				ret = sock_intr_errno(timeo);
730 				break;
731 			}
732 			continue;
733 		}
734 		tss.len -= ret;
735 		spliced += ret;
736 
737 		if (!timeo)
738 			break;
739 		release_sock(sk);
740 		lock_sock(sk);
741 
742 		if (sk->sk_err || sk->sk_state == TCP_CLOSE ||
743 		    (sk->sk_shutdown & RCV_SHUTDOWN) ||
744 		    signal_pending(current))
745 			break;
746 	}
747 
748 	release_sock(sk);
749 
750 	if (spliced)
751 		return spliced;
752 
753 	return ret;
754 }
755 EXPORT_SYMBOL(tcp_splice_read);
756 
757 struct sk_buff *sk_stream_alloc_skb(struct sock *sk, int size, gfp_t gfp)
758 {
759 	struct sk_buff *skb;
760 
761 	/* The TCP header must be at least 32-bit aligned.  */
762 	size = ALIGN(size, 4);
763 
764 	skb = alloc_skb_fclone(size + sk->sk_prot->max_header, gfp);
765 	if (skb) {
766 		if (sk_wmem_schedule(sk, skb->truesize)) {
767 			skb_reserve(skb, sk->sk_prot->max_header);
768 			/*
769 			 * Make sure that we have exactly size bytes
770 			 * available to the caller, no more, no less.
771 			 */
772 			skb->reserved_tailroom = skb->end - skb->tail - size;
773 			return skb;
774 		}
775 		__kfree_skb(skb);
776 	} else {
777 		sk->sk_prot->enter_memory_pressure(sk);
778 		sk_stream_moderate_sndbuf(sk);
779 	}
780 	return NULL;
781 }
782 
783 static unsigned int tcp_xmit_size_goal(struct sock *sk, u32 mss_now,
784 				       int large_allowed)
785 {
786 	struct tcp_sock *tp = tcp_sk(sk);
787 	u32 xmit_size_goal, old_size_goal;
788 
789 	xmit_size_goal = mss_now;
790 
791 	if (large_allowed && sk_can_gso(sk)) {
792 		xmit_size_goal = ((sk->sk_gso_max_size - 1) -
793 				  inet_csk(sk)->icsk_af_ops->net_header_len -
794 				  inet_csk(sk)->icsk_ext_hdr_len -
795 				  tp->tcp_header_len);
796 
797 		/* TSQ : try to have two TSO segments in flight */
798 		xmit_size_goal = min_t(u32, xmit_size_goal,
799 				       sysctl_tcp_limit_output_bytes >> 1);
800 
801 		xmit_size_goal = tcp_bound_to_half_wnd(tp, xmit_size_goal);
802 
803 		/* We try hard to avoid divides here */
804 		old_size_goal = tp->xmit_size_goal_segs * mss_now;
805 
806 		if (likely(old_size_goal <= xmit_size_goal &&
807 			   old_size_goal + mss_now > xmit_size_goal)) {
808 			xmit_size_goal = old_size_goal;
809 		} else {
810 			tp->xmit_size_goal_segs =
811 				min_t(u16, xmit_size_goal / mss_now,
812 				      sk->sk_gso_max_segs);
813 			xmit_size_goal = tp->xmit_size_goal_segs * mss_now;
814 		}
815 	}
816 
817 	return max(xmit_size_goal, mss_now);
818 }
819 
820 static int tcp_send_mss(struct sock *sk, int *size_goal, int flags)
821 {
822 	int mss_now;
823 
824 	mss_now = tcp_current_mss(sk);
825 	*size_goal = tcp_xmit_size_goal(sk, mss_now, !(flags & MSG_OOB));
826 
827 	return mss_now;
828 }
829 
830 static ssize_t do_tcp_sendpages(struct sock *sk, struct page *page, int offset,
831 				size_t size, int flags)
832 {
833 	struct tcp_sock *tp = tcp_sk(sk);
834 	int mss_now, size_goal;
835 	int err;
836 	ssize_t copied;
837 	long timeo = sock_sndtimeo(sk, flags & MSG_DONTWAIT);
838 
839 	/* Wait for a connection to finish. One exception is TCP Fast Open
840 	 * (passive side) where data is allowed to be sent before a connection
841 	 * is fully established.
842 	 */
843 	if (((1 << sk->sk_state) & ~(TCPF_ESTABLISHED | TCPF_CLOSE_WAIT)) &&
844 	    !tcp_passive_fastopen(sk)) {
845 		if ((err = sk_stream_wait_connect(sk, &timeo)) != 0)
846 			goto out_err;
847 	}
848 
849 	clear_bit(SOCK_ASYNC_NOSPACE, &sk->sk_socket->flags);
850 
851 	mss_now = tcp_send_mss(sk, &size_goal, flags);
852 	copied = 0;
853 
854 	err = -EPIPE;
855 	if (sk->sk_err || (sk->sk_shutdown & SEND_SHUTDOWN))
856 		goto out_err;
857 
858 	while (size > 0) {
859 		struct sk_buff *skb = tcp_write_queue_tail(sk);
860 		int copy, i;
861 		bool can_coalesce;
862 
863 		if (!tcp_send_head(sk) || (copy = size_goal - skb->len) <= 0) {
864 new_segment:
865 			if (!sk_stream_memory_free(sk))
866 				goto wait_for_sndbuf;
867 
868 			skb = sk_stream_alloc_skb(sk, 0, sk->sk_allocation);
869 			if (!skb)
870 				goto wait_for_memory;
871 
872 			skb_entail(sk, skb);
873 			copy = size_goal;
874 		}
875 
876 		if (copy > size)
877 			copy = size;
878 
879 		i = skb_shinfo(skb)->nr_frags;
880 		can_coalesce = skb_can_coalesce(skb, i, page, offset);
881 		if (!can_coalesce && i >= MAX_SKB_FRAGS) {
882 			tcp_mark_push(tp, skb);
883 			goto new_segment;
884 		}
885 		if (!sk_wmem_schedule(sk, copy))
886 			goto wait_for_memory;
887 
888 		if (can_coalesce) {
889 			skb_frag_size_add(&skb_shinfo(skb)->frags[i - 1], copy);
890 		} else {
891 			get_page(page);
892 			skb_fill_page_desc(skb, i, page, offset, copy);
893 		}
894 		skb_shinfo(skb)->tx_flags |= SKBTX_SHARED_FRAG;
895 
896 		skb->len += copy;
897 		skb->data_len += copy;
898 		skb->truesize += copy;
899 		sk->sk_wmem_queued += copy;
900 		sk_mem_charge(sk, copy);
901 		skb->ip_summed = CHECKSUM_PARTIAL;
902 		tp->write_seq += copy;
903 		TCP_SKB_CB(skb)->end_seq += copy;
904 		skb_shinfo(skb)->gso_segs = 0;
905 
906 		if (!copied)
907 			TCP_SKB_CB(skb)->tcp_flags &= ~TCPHDR_PSH;
908 
909 		copied += copy;
910 		offset += copy;
911 		if (!(size -= copy))
912 			goto out;
913 
914 		if (skb->len < size_goal || (flags & MSG_OOB))
915 			continue;
916 
917 		if (forced_push(tp)) {
918 			tcp_mark_push(tp, skb);
919 			__tcp_push_pending_frames(sk, mss_now, TCP_NAGLE_PUSH);
920 		} else if (skb == tcp_send_head(sk))
921 			tcp_push_one(sk, mss_now);
922 		continue;
923 
924 wait_for_sndbuf:
925 		set_bit(SOCK_NOSPACE, &sk->sk_socket->flags);
926 wait_for_memory:
927 		tcp_push(sk, flags & ~MSG_MORE, mss_now, TCP_NAGLE_PUSH);
928 
929 		if ((err = sk_stream_wait_memory(sk, &timeo)) != 0)
930 			goto do_error;
931 
932 		mss_now = tcp_send_mss(sk, &size_goal, flags);
933 	}
934 
935 out:
936 	if (copied && !(flags & MSG_SENDPAGE_NOTLAST))
937 		tcp_push(sk, flags, mss_now, tp->nonagle);
938 	return copied;
939 
940 do_error:
941 	if (copied)
942 		goto out;
943 out_err:
944 	return sk_stream_error(sk, flags, err);
945 }
946 
947 int tcp_sendpage(struct sock *sk, struct page *page, int offset,
948 		 size_t size, int flags)
949 {
950 	ssize_t res;
951 
952 	if (!(sk->sk_route_caps & NETIF_F_SG) ||
953 	    !(sk->sk_route_caps & NETIF_F_ALL_CSUM))
954 		return sock_no_sendpage(sk->sk_socket, page, offset, size,
955 					flags);
956 
957 	lock_sock(sk);
958 	res = do_tcp_sendpages(sk, page, offset, size, flags);
959 	release_sock(sk);
960 	return res;
961 }
962 EXPORT_SYMBOL(tcp_sendpage);
963 
964 static inline int select_size(const struct sock *sk, bool sg)
965 {
966 	const struct tcp_sock *tp = tcp_sk(sk);
967 	int tmp = tp->mss_cache;
968 
969 	if (sg) {
970 		if (sk_can_gso(sk)) {
971 			/* Small frames wont use a full page:
972 			 * Payload will immediately follow tcp header.
973 			 */
974 			tmp = SKB_WITH_OVERHEAD(2048 - MAX_TCP_HEADER);
975 		} else {
976 			int pgbreak = SKB_MAX_HEAD(MAX_TCP_HEADER);
977 
978 			if (tmp >= pgbreak &&
979 			    tmp <= pgbreak + (MAX_SKB_FRAGS - 1) * PAGE_SIZE)
980 				tmp = pgbreak;
981 		}
982 	}
983 
984 	return tmp;
985 }
986 
987 void tcp_free_fastopen_req(struct tcp_sock *tp)
988 {
989 	if (tp->fastopen_req != NULL) {
990 		kfree(tp->fastopen_req);
991 		tp->fastopen_req = NULL;
992 	}
993 }
994 
995 static int tcp_sendmsg_fastopen(struct sock *sk, struct msghdr *msg, int *size)
996 {
997 	struct tcp_sock *tp = tcp_sk(sk);
998 	int err, flags;
999 
1000 	if (!(sysctl_tcp_fastopen & TFO_CLIENT_ENABLE))
1001 		return -EOPNOTSUPP;
1002 	if (tp->fastopen_req != NULL)
1003 		return -EALREADY; /* Another Fast Open is in progress */
1004 
1005 	tp->fastopen_req = kzalloc(sizeof(struct tcp_fastopen_request),
1006 				   sk->sk_allocation);
1007 	if (unlikely(tp->fastopen_req == NULL))
1008 		return -ENOBUFS;
1009 	tp->fastopen_req->data = msg;
1010 
1011 	flags = (msg->msg_flags & MSG_DONTWAIT) ? O_NONBLOCK : 0;
1012 	err = __inet_stream_connect(sk->sk_socket, msg->msg_name,
1013 				    msg->msg_namelen, flags);
1014 	*size = tp->fastopen_req->copied;
1015 	tcp_free_fastopen_req(tp);
1016 	return err;
1017 }
1018 
1019 int tcp_sendmsg(struct kiocb *iocb, struct sock *sk, struct msghdr *msg,
1020 		size_t size)
1021 {
1022 	struct iovec *iov;
1023 	struct tcp_sock *tp = tcp_sk(sk);
1024 	struct sk_buff *skb;
1025 	int iovlen, flags, err, copied = 0;
1026 	int mss_now = 0, size_goal, copied_syn = 0, offset = 0;
1027 	bool sg;
1028 	long timeo;
1029 
1030 	lock_sock(sk);
1031 
1032 	flags = msg->msg_flags;
1033 	if (flags & MSG_FASTOPEN) {
1034 		err = tcp_sendmsg_fastopen(sk, msg, &copied_syn);
1035 		if (err == -EINPROGRESS && copied_syn > 0)
1036 			goto out;
1037 		else if (err)
1038 			goto out_err;
1039 		offset = copied_syn;
1040 	}
1041 
1042 	timeo = sock_sndtimeo(sk, flags & MSG_DONTWAIT);
1043 
1044 	/* Wait for a connection to finish. One exception is TCP Fast Open
1045 	 * (passive side) where data is allowed to be sent before a connection
1046 	 * is fully established.
1047 	 */
1048 	if (((1 << sk->sk_state) & ~(TCPF_ESTABLISHED | TCPF_CLOSE_WAIT)) &&
1049 	    !tcp_passive_fastopen(sk)) {
1050 		if ((err = sk_stream_wait_connect(sk, &timeo)) != 0)
1051 			goto do_error;
1052 	}
1053 
1054 	if (unlikely(tp->repair)) {
1055 		if (tp->repair_queue == TCP_RECV_QUEUE) {
1056 			copied = tcp_send_rcvq(sk, msg, size);
1057 			goto out;
1058 		}
1059 
1060 		err = -EINVAL;
1061 		if (tp->repair_queue == TCP_NO_QUEUE)
1062 			goto out_err;
1063 
1064 		/* 'common' sending to sendq */
1065 	}
1066 
1067 	/* This should be in poll */
1068 	clear_bit(SOCK_ASYNC_NOSPACE, &sk->sk_socket->flags);
1069 
1070 	mss_now = tcp_send_mss(sk, &size_goal, flags);
1071 
1072 	/* Ok commence sending. */
1073 	iovlen = msg->msg_iovlen;
1074 	iov = msg->msg_iov;
1075 	copied = 0;
1076 
1077 	err = -EPIPE;
1078 	if (sk->sk_err || (sk->sk_shutdown & SEND_SHUTDOWN))
1079 		goto out_err;
1080 
1081 	sg = !!(sk->sk_route_caps & NETIF_F_SG);
1082 
1083 	while (--iovlen >= 0) {
1084 		size_t seglen = iov->iov_len;
1085 		unsigned char __user *from = iov->iov_base;
1086 
1087 		iov++;
1088 		if (unlikely(offset > 0)) {  /* Skip bytes copied in SYN */
1089 			if (offset >= seglen) {
1090 				offset -= seglen;
1091 				continue;
1092 			}
1093 			seglen -= offset;
1094 			from += offset;
1095 			offset = 0;
1096 		}
1097 
1098 		while (seglen > 0) {
1099 			int copy = 0;
1100 			int max = size_goal;
1101 
1102 			skb = tcp_write_queue_tail(sk);
1103 			if (tcp_send_head(sk)) {
1104 				if (skb->ip_summed == CHECKSUM_NONE)
1105 					max = mss_now;
1106 				copy = max - skb->len;
1107 			}
1108 
1109 			if (copy <= 0) {
1110 new_segment:
1111 				/* Allocate new segment. If the interface is SG,
1112 				 * allocate skb fitting to single page.
1113 				 */
1114 				if (!sk_stream_memory_free(sk))
1115 					goto wait_for_sndbuf;
1116 
1117 				skb = sk_stream_alloc_skb(sk,
1118 							  select_size(sk, sg),
1119 							  sk->sk_allocation);
1120 				if (!skb)
1121 					goto wait_for_memory;
1122 
1123 				/*
1124 				 * Check whether we can use HW checksum.
1125 				 */
1126 				if (sk->sk_route_caps & NETIF_F_ALL_CSUM)
1127 					skb->ip_summed = CHECKSUM_PARTIAL;
1128 
1129 				skb_entail(sk, skb);
1130 				copy = size_goal;
1131 				max = size_goal;
1132 			}
1133 
1134 			/* Try to append data to the end of skb. */
1135 			if (copy > seglen)
1136 				copy = seglen;
1137 
1138 			/* Where to copy to? */
1139 			if (skb_availroom(skb) > 0) {
1140 				/* We have some space in skb head. Superb! */
1141 				copy = min_t(int, copy, skb_availroom(skb));
1142 				err = skb_add_data_nocache(sk, skb, from, copy);
1143 				if (err)
1144 					goto do_fault;
1145 			} else {
1146 				bool merge = true;
1147 				int i = skb_shinfo(skb)->nr_frags;
1148 				struct page_frag *pfrag = sk_page_frag(sk);
1149 
1150 				if (!sk_page_frag_refill(sk, pfrag))
1151 					goto wait_for_memory;
1152 
1153 				if (!skb_can_coalesce(skb, i, pfrag->page,
1154 						      pfrag->offset)) {
1155 					if (i == MAX_SKB_FRAGS || !sg) {
1156 						tcp_mark_push(tp, skb);
1157 						goto new_segment;
1158 					}
1159 					merge = false;
1160 				}
1161 
1162 				copy = min_t(int, copy, pfrag->size - pfrag->offset);
1163 
1164 				if (!sk_wmem_schedule(sk, copy))
1165 					goto wait_for_memory;
1166 
1167 				err = skb_copy_to_page_nocache(sk, from, skb,
1168 							       pfrag->page,
1169 							       pfrag->offset,
1170 							       copy);
1171 				if (err)
1172 					goto do_error;
1173 
1174 				/* Update the skb. */
1175 				if (merge) {
1176 					skb_frag_size_add(&skb_shinfo(skb)->frags[i - 1], copy);
1177 				} else {
1178 					skb_fill_page_desc(skb, i, pfrag->page,
1179 							   pfrag->offset, copy);
1180 					get_page(pfrag->page);
1181 				}
1182 				pfrag->offset += copy;
1183 			}
1184 
1185 			if (!copied)
1186 				TCP_SKB_CB(skb)->tcp_flags &= ~TCPHDR_PSH;
1187 
1188 			tp->write_seq += copy;
1189 			TCP_SKB_CB(skb)->end_seq += copy;
1190 			skb_shinfo(skb)->gso_segs = 0;
1191 
1192 			from += copy;
1193 			copied += copy;
1194 			if ((seglen -= copy) == 0 && iovlen == 0)
1195 				goto out;
1196 
1197 			if (skb->len < max || (flags & MSG_OOB) || unlikely(tp->repair))
1198 				continue;
1199 
1200 			if (forced_push(tp)) {
1201 				tcp_mark_push(tp, skb);
1202 				__tcp_push_pending_frames(sk, mss_now, TCP_NAGLE_PUSH);
1203 			} else if (skb == tcp_send_head(sk))
1204 				tcp_push_one(sk, mss_now);
1205 			continue;
1206 
1207 wait_for_sndbuf:
1208 			set_bit(SOCK_NOSPACE, &sk->sk_socket->flags);
1209 wait_for_memory:
1210 			if (copied)
1211 				tcp_push(sk, flags & ~MSG_MORE, mss_now, TCP_NAGLE_PUSH);
1212 
1213 			if ((err = sk_stream_wait_memory(sk, &timeo)) != 0)
1214 				goto do_error;
1215 
1216 			mss_now = tcp_send_mss(sk, &size_goal, flags);
1217 		}
1218 	}
1219 
1220 out:
1221 	if (copied)
1222 		tcp_push(sk, flags, mss_now, tp->nonagle);
1223 	release_sock(sk);
1224 	return copied + copied_syn;
1225 
1226 do_fault:
1227 	if (!skb->len) {
1228 		tcp_unlink_write_queue(skb, sk);
1229 		/* It is the one place in all of TCP, except connection
1230 		 * reset, where we can be unlinking the send_head.
1231 		 */
1232 		tcp_check_send_head(sk, skb);
1233 		sk_wmem_free_skb(sk, skb);
1234 	}
1235 
1236 do_error:
1237 	if (copied + copied_syn)
1238 		goto out;
1239 out_err:
1240 	err = sk_stream_error(sk, flags, err);
1241 	release_sock(sk);
1242 	return err;
1243 }
1244 EXPORT_SYMBOL(tcp_sendmsg);
1245 
1246 /*
1247  *	Handle reading urgent data. BSD has very simple semantics for
1248  *	this, no blocking and very strange errors 8)
1249  */
1250 
1251 static int tcp_recv_urg(struct sock *sk, struct msghdr *msg, int len, int flags)
1252 {
1253 	struct tcp_sock *tp = tcp_sk(sk);
1254 
1255 	/* No URG data to read. */
1256 	if (sock_flag(sk, SOCK_URGINLINE) || !tp->urg_data ||
1257 	    tp->urg_data == TCP_URG_READ)
1258 		return -EINVAL;	/* Yes this is right ! */
1259 
1260 	if (sk->sk_state == TCP_CLOSE && !sock_flag(sk, SOCK_DONE))
1261 		return -ENOTCONN;
1262 
1263 	if (tp->urg_data & TCP_URG_VALID) {
1264 		int err = 0;
1265 		char c = tp->urg_data;
1266 
1267 		if (!(flags & MSG_PEEK))
1268 			tp->urg_data = TCP_URG_READ;
1269 
1270 		/* Read urgent data. */
1271 		msg->msg_flags |= MSG_OOB;
1272 
1273 		if (len > 0) {
1274 			if (!(flags & MSG_TRUNC))
1275 				err = memcpy_toiovec(msg->msg_iov, &c, 1);
1276 			len = 1;
1277 		} else
1278 			msg->msg_flags |= MSG_TRUNC;
1279 
1280 		return err ? -EFAULT : len;
1281 	}
1282 
1283 	if (sk->sk_state == TCP_CLOSE || (sk->sk_shutdown & RCV_SHUTDOWN))
1284 		return 0;
1285 
1286 	/* Fixed the recv(..., MSG_OOB) behaviour.  BSD docs and
1287 	 * the available implementations agree in this case:
1288 	 * this call should never block, independent of the
1289 	 * blocking state of the socket.
1290 	 * Mike <pall@rz.uni-karlsruhe.de>
1291 	 */
1292 	return -EAGAIN;
1293 }
1294 
1295 static int tcp_peek_sndq(struct sock *sk, struct msghdr *msg, int len)
1296 {
1297 	struct sk_buff *skb;
1298 	int copied = 0, err = 0;
1299 
1300 	/* XXX -- need to support SO_PEEK_OFF */
1301 
1302 	skb_queue_walk(&sk->sk_write_queue, skb) {
1303 		err = skb_copy_datagram_iovec(skb, 0, msg->msg_iov, skb->len);
1304 		if (err)
1305 			break;
1306 
1307 		copied += skb->len;
1308 	}
1309 
1310 	return err ?: copied;
1311 }
1312 
1313 /* Clean up the receive buffer for full frames taken by the user,
1314  * then send an ACK if necessary.  COPIED is the number of bytes
1315  * tcp_recvmsg has given to the user so far, it speeds up the
1316  * calculation of whether or not we must ACK for the sake of
1317  * a window update.
1318  */
1319 void tcp_cleanup_rbuf(struct sock *sk, int copied)
1320 {
1321 	struct tcp_sock *tp = tcp_sk(sk);
1322 	bool time_to_ack = false;
1323 
1324 	struct sk_buff *skb = skb_peek(&sk->sk_receive_queue);
1325 
1326 	WARN(skb && !before(tp->copied_seq, TCP_SKB_CB(skb)->end_seq),
1327 	     "cleanup rbuf bug: copied %X seq %X rcvnxt %X\n",
1328 	     tp->copied_seq, TCP_SKB_CB(skb)->end_seq, tp->rcv_nxt);
1329 
1330 	if (inet_csk_ack_scheduled(sk)) {
1331 		const struct inet_connection_sock *icsk = inet_csk(sk);
1332 		   /* Delayed ACKs frequently hit locked sockets during bulk
1333 		    * receive. */
1334 		if (icsk->icsk_ack.blocked ||
1335 		    /* Once-per-two-segments ACK was not sent by tcp_input.c */
1336 		    tp->rcv_nxt - tp->rcv_wup > icsk->icsk_ack.rcv_mss ||
1337 		    /*
1338 		     * If this read emptied read buffer, we send ACK, if
1339 		     * connection is not bidirectional, user drained
1340 		     * receive buffer and there was a small segment
1341 		     * in queue.
1342 		     */
1343 		    (copied > 0 &&
1344 		     ((icsk->icsk_ack.pending & ICSK_ACK_PUSHED2) ||
1345 		      ((icsk->icsk_ack.pending & ICSK_ACK_PUSHED) &&
1346 		       !icsk->icsk_ack.pingpong)) &&
1347 		      !atomic_read(&sk->sk_rmem_alloc)))
1348 			time_to_ack = true;
1349 	}
1350 
1351 	/* We send an ACK if we can now advertise a non-zero window
1352 	 * which has been raised "significantly".
1353 	 *
1354 	 * Even if window raised up to infinity, do not send window open ACK
1355 	 * in states, where we will not receive more. It is useless.
1356 	 */
1357 	if (copied > 0 && !time_to_ack && !(sk->sk_shutdown & RCV_SHUTDOWN)) {
1358 		__u32 rcv_window_now = tcp_receive_window(tp);
1359 
1360 		/* Optimize, __tcp_select_window() is not cheap. */
1361 		if (2*rcv_window_now <= tp->window_clamp) {
1362 			__u32 new_window = __tcp_select_window(sk);
1363 
1364 			/* Send ACK now, if this read freed lots of space
1365 			 * in our buffer. Certainly, new_window is new window.
1366 			 * We can advertise it now, if it is not less than current one.
1367 			 * "Lots" means "at least twice" here.
1368 			 */
1369 			if (new_window && new_window >= 2 * rcv_window_now)
1370 				time_to_ack = true;
1371 		}
1372 	}
1373 	if (time_to_ack)
1374 		tcp_send_ack(sk);
1375 }
1376 
1377 static void tcp_prequeue_process(struct sock *sk)
1378 {
1379 	struct sk_buff *skb;
1380 	struct tcp_sock *tp = tcp_sk(sk);
1381 
1382 	NET_INC_STATS_USER(sock_net(sk), LINUX_MIB_TCPPREQUEUED);
1383 
1384 	/* RX process wants to run with disabled BHs, though it is not
1385 	 * necessary */
1386 	local_bh_disable();
1387 	while ((skb = __skb_dequeue(&tp->ucopy.prequeue)) != NULL)
1388 		sk_backlog_rcv(sk, skb);
1389 	local_bh_enable();
1390 
1391 	/* Clear memory counter. */
1392 	tp->ucopy.memory = 0;
1393 }
1394 
1395 #ifdef CONFIG_NET_DMA
1396 static void tcp_service_net_dma(struct sock *sk, bool wait)
1397 {
1398 	dma_cookie_t done, used;
1399 	dma_cookie_t last_issued;
1400 	struct tcp_sock *tp = tcp_sk(sk);
1401 
1402 	if (!tp->ucopy.dma_chan)
1403 		return;
1404 
1405 	last_issued = tp->ucopy.dma_cookie;
1406 	dma_async_issue_pending(tp->ucopy.dma_chan);
1407 
1408 	do {
1409 		if (dma_async_is_tx_complete(tp->ucopy.dma_chan,
1410 					      last_issued, &done,
1411 					      &used) == DMA_SUCCESS) {
1412 			/* Safe to free early-copied skbs now */
1413 			__skb_queue_purge(&sk->sk_async_wait_queue);
1414 			break;
1415 		} else {
1416 			struct sk_buff *skb;
1417 			while ((skb = skb_peek(&sk->sk_async_wait_queue)) &&
1418 			       (dma_async_is_complete(skb->dma_cookie, done,
1419 						      used) == DMA_SUCCESS)) {
1420 				__skb_dequeue(&sk->sk_async_wait_queue);
1421 				kfree_skb(skb);
1422 			}
1423 		}
1424 	} while (wait);
1425 }
1426 #endif
1427 
1428 static struct sk_buff *tcp_recv_skb(struct sock *sk, u32 seq, u32 *off)
1429 {
1430 	struct sk_buff *skb;
1431 	u32 offset;
1432 
1433 	while ((skb = skb_peek(&sk->sk_receive_queue)) != NULL) {
1434 		offset = seq - TCP_SKB_CB(skb)->seq;
1435 		if (tcp_hdr(skb)->syn)
1436 			offset--;
1437 		if (offset < skb->len || tcp_hdr(skb)->fin) {
1438 			*off = offset;
1439 			return skb;
1440 		}
1441 		/* This looks weird, but this can happen if TCP collapsing
1442 		 * splitted a fat GRO packet, while we released socket lock
1443 		 * in skb_splice_bits()
1444 		 */
1445 		sk_eat_skb(sk, skb, false);
1446 	}
1447 	return NULL;
1448 }
1449 
1450 /*
1451  * This routine provides an alternative to tcp_recvmsg() for routines
1452  * that would like to handle copying from skbuffs directly in 'sendfile'
1453  * fashion.
1454  * Note:
1455  *	- It is assumed that the socket was locked by the caller.
1456  *	- The routine does not block.
1457  *	- At present, there is no support for reading OOB data
1458  *	  or for 'peeking' the socket using this routine
1459  *	  (although both would be easy to implement).
1460  */
1461 int tcp_read_sock(struct sock *sk, read_descriptor_t *desc,
1462 		  sk_read_actor_t recv_actor)
1463 {
1464 	struct sk_buff *skb;
1465 	struct tcp_sock *tp = tcp_sk(sk);
1466 	u32 seq = tp->copied_seq;
1467 	u32 offset;
1468 	int copied = 0;
1469 
1470 	if (sk->sk_state == TCP_LISTEN)
1471 		return -ENOTCONN;
1472 	while ((skb = tcp_recv_skb(sk, seq, &offset)) != NULL) {
1473 		if (offset < skb->len) {
1474 			int used;
1475 			size_t len;
1476 
1477 			len = skb->len - offset;
1478 			/* Stop reading if we hit a patch of urgent data */
1479 			if (tp->urg_data) {
1480 				u32 urg_offset = tp->urg_seq - seq;
1481 				if (urg_offset < len)
1482 					len = urg_offset;
1483 				if (!len)
1484 					break;
1485 			}
1486 			used = recv_actor(desc, skb, offset, len);
1487 			if (used <= 0) {
1488 				if (!copied)
1489 					copied = used;
1490 				break;
1491 			} else if (used <= len) {
1492 				seq += used;
1493 				copied += used;
1494 				offset += used;
1495 			}
1496 			/* If recv_actor drops the lock (e.g. TCP splice
1497 			 * receive) the skb pointer might be invalid when
1498 			 * getting here: tcp_collapse might have deleted it
1499 			 * while aggregating skbs from the socket queue.
1500 			 */
1501 			skb = tcp_recv_skb(sk, seq - 1, &offset);
1502 			if (!skb)
1503 				break;
1504 			/* TCP coalescing might have appended data to the skb.
1505 			 * Try to splice more frags
1506 			 */
1507 			if (offset + 1 != skb->len)
1508 				continue;
1509 		}
1510 		if (tcp_hdr(skb)->fin) {
1511 			sk_eat_skb(sk, skb, false);
1512 			++seq;
1513 			break;
1514 		}
1515 		sk_eat_skb(sk, skb, false);
1516 		if (!desc->count)
1517 			break;
1518 		tp->copied_seq = seq;
1519 	}
1520 	tp->copied_seq = seq;
1521 
1522 	tcp_rcv_space_adjust(sk);
1523 
1524 	/* Clean up data we have read: This will do ACK frames. */
1525 	if (copied > 0) {
1526 		tcp_recv_skb(sk, seq, &offset);
1527 		tcp_cleanup_rbuf(sk, copied);
1528 	}
1529 	return copied;
1530 }
1531 EXPORT_SYMBOL(tcp_read_sock);
1532 
1533 /*
1534  *	This routine copies from a sock struct into the user buffer.
1535  *
1536  *	Technical note: in 2.3 we work on _locked_ socket, so that
1537  *	tricks with *seq access order and skb->users are not required.
1538  *	Probably, code can be easily improved even more.
1539  */
1540 
1541 int tcp_recvmsg(struct kiocb *iocb, struct sock *sk, struct msghdr *msg,
1542 		size_t len, int nonblock, int flags, int *addr_len)
1543 {
1544 	struct tcp_sock *tp = tcp_sk(sk);
1545 	int copied = 0;
1546 	u32 peek_seq;
1547 	u32 *seq;
1548 	unsigned long used;
1549 	int err;
1550 	int target;		/* Read at least this many bytes */
1551 	long timeo;
1552 	struct task_struct *user_recv = NULL;
1553 	bool copied_early = false;
1554 	struct sk_buff *skb;
1555 	u32 urg_hole = 0;
1556 
1557 	if (sk_can_busy_loop(sk) && skb_queue_empty(&sk->sk_receive_queue) &&
1558 	    (sk->sk_state == TCP_ESTABLISHED))
1559 		sk_busy_loop(sk, nonblock);
1560 
1561 	lock_sock(sk);
1562 
1563 	err = -ENOTCONN;
1564 	if (sk->sk_state == TCP_LISTEN)
1565 		goto out;
1566 
1567 	timeo = sock_rcvtimeo(sk, nonblock);
1568 
1569 	/* Urgent data needs to be handled specially. */
1570 	if (flags & MSG_OOB)
1571 		goto recv_urg;
1572 
1573 	if (unlikely(tp->repair)) {
1574 		err = -EPERM;
1575 		if (!(flags & MSG_PEEK))
1576 			goto out;
1577 
1578 		if (tp->repair_queue == TCP_SEND_QUEUE)
1579 			goto recv_sndq;
1580 
1581 		err = -EINVAL;
1582 		if (tp->repair_queue == TCP_NO_QUEUE)
1583 			goto out;
1584 
1585 		/* 'common' recv queue MSG_PEEK-ing */
1586 	}
1587 
1588 	seq = &tp->copied_seq;
1589 	if (flags & MSG_PEEK) {
1590 		peek_seq = tp->copied_seq;
1591 		seq = &peek_seq;
1592 	}
1593 
1594 	target = sock_rcvlowat(sk, flags & MSG_WAITALL, len);
1595 
1596 #ifdef CONFIG_NET_DMA
1597 	tp->ucopy.dma_chan = NULL;
1598 	preempt_disable();
1599 	skb = skb_peek_tail(&sk->sk_receive_queue);
1600 	{
1601 		int available = 0;
1602 
1603 		if (skb)
1604 			available = TCP_SKB_CB(skb)->seq + skb->len - (*seq);
1605 		if ((available < target) &&
1606 		    (len > sysctl_tcp_dma_copybreak) && !(flags & MSG_PEEK) &&
1607 		    !sysctl_tcp_low_latency &&
1608 		    net_dma_find_channel()) {
1609 			preempt_enable_no_resched();
1610 			tp->ucopy.pinned_list =
1611 					dma_pin_iovec_pages(msg->msg_iov, len);
1612 		} else {
1613 			preempt_enable_no_resched();
1614 		}
1615 	}
1616 #endif
1617 
1618 	do {
1619 		u32 offset;
1620 
1621 		/* Are we at urgent data? Stop if we have read anything or have SIGURG pending. */
1622 		if (tp->urg_data && tp->urg_seq == *seq) {
1623 			if (copied)
1624 				break;
1625 			if (signal_pending(current)) {
1626 				copied = timeo ? sock_intr_errno(timeo) : -EAGAIN;
1627 				break;
1628 			}
1629 		}
1630 
1631 		/* Next get a buffer. */
1632 
1633 		skb_queue_walk(&sk->sk_receive_queue, skb) {
1634 			/* Now that we have two receive queues this
1635 			 * shouldn't happen.
1636 			 */
1637 			if (WARN(before(*seq, TCP_SKB_CB(skb)->seq),
1638 				 "recvmsg bug: copied %X seq %X rcvnxt %X fl %X\n",
1639 				 *seq, TCP_SKB_CB(skb)->seq, tp->rcv_nxt,
1640 				 flags))
1641 				break;
1642 
1643 			offset = *seq - TCP_SKB_CB(skb)->seq;
1644 			if (tcp_hdr(skb)->syn)
1645 				offset--;
1646 			if (offset < skb->len)
1647 				goto found_ok_skb;
1648 			if (tcp_hdr(skb)->fin)
1649 				goto found_fin_ok;
1650 			WARN(!(flags & MSG_PEEK),
1651 			     "recvmsg bug 2: copied %X seq %X rcvnxt %X fl %X\n",
1652 			     *seq, TCP_SKB_CB(skb)->seq, tp->rcv_nxt, flags);
1653 		}
1654 
1655 		/* Well, if we have backlog, try to process it now yet. */
1656 
1657 		if (copied >= target && !sk->sk_backlog.tail)
1658 			break;
1659 
1660 		if (copied) {
1661 			if (sk->sk_err ||
1662 			    sk->sk_state == TCP_CLOSE ||
1663 			    (sk->sk_shutdown & RCV_SHUTDOWN) ||
1664 			    !timeo ||
1665 			    signal_pending(current))
1666 				break;
1667 		} else {
1668 			if (sock_flag(sk, SOCK_DONE))
1669 				break;
1670 
1671 			if (sk->sk_err) {
1672 				copied = sock_error(sk);
1673 				break;
1674 			}
1675 
1676 			if (sk->sk_shutdown & RCV_SHUTDOWN)
1677 				break;
1678 
1679 			if (sk->sk_state == TCP_CLOSE) {
1680 				if (!sock_flag(sk, SOCK_DONE)) {
1681 					/* This occurs when user tries to read
1682 					 * from never connected socket.
1683 					 */
1684 					copied = -ENOTCONN;
1685 					break;
1686 				}
1687 				break;
1688 			}
1689 
1690 			if (!timeo) {
1691 				copied = -EAGAIN;
1692 				break;
1693 			}
1694 
1695 			if (signal_pending(current)) {
1696 				copied = sock_intr_errno(timeo);
1697 				break;
1698 			}
1699 		}
1700 
1701 		tcp_cleanup_rbuf(sk, copied);
1702 
1703 		if (!sysctl_tcp_low_latency && tp->ucopy.task == user_recv) {
1704 			/* Install new reader */
1705 			if (!user_recv && !(flags & (MSG_TRUNC | MSG_PEEK))) {
1706 				user_recv = current;
1707 				tp->ucopy.task = user_recv;
1708 				tp->ucopy.iov = msg->msg_iov;
1709 			}
1710 
1711 			tp->ucopy.len = len;
1712 
1713 			WARN_ON(tp->copied_seq != tp->rcv_nxt &&
1714 				!(flags & (MSG_PEEK | MSG_TRUNC)));
1715 
1716 			/* Ugly... If prequeue is not empty, we have to
1717 			 * process it before releasing socket, otherwise
1718 			 * order will be broken at second iteration.
1719 			 * More elegant solution is required!!!
1720 			 *
1721 			 * Look: we have the following (pseudo)queues:
1722 			 *
1723 			 * 1. packets in flight
1724 			 * 2. backlog
1725 			 * 3. prequeue
1726 			 * 4. receive_queue
1727 			 *
1728 			 * Each queue can be processed only if the next ones
1729 			 * are empty. At this point we have empty receive_queue.
1730 			 * But prequeue _can_ be not empty after 2nd iteration,
1731 			 * when we jumped to start of loop because backlog
1732 			 * processing added something to receive_queue.
1733 			 * We cannot release_sock(), because backlog contains
1734 			 * packets arrived _after_ prequeued ones.
1735 			 *
1736 			 * Shortly, algorithm is clear --- to process all
1737 			 * the queues in order. We could make it more directly,
1738 			 * requeueing packets from backlog to prequeue, if
1739 			 * is not empty. It is more elegant, but eats cycles,
1740 			 * unfortunately.
1741 			 */
1742 			if (!skb_queue_empty(&tp->ucopy.prequeue))
1743 				goto do_prequeue;
1744 
1745 			/* __ Set realtime policy in scheduler __ */
1746 		}
1747 
1748 #ifdef CONFIG_NET_DMA
1749 		if (tp->ucopy.dma_chan) {
1750 			if (tp->rcv_wnd == 0 &&
1751 			    !skb_queue_empty(&sk->sk_async_wait_queue)) {
1752 				tcp_service_net_dma(sk, true);
1753 				tcp_cleanup_rbuf(sk, copied);
1754 			} else
1755 				dma_async_issue_pending(tp->ucopy.dma_chan);
1756 		}
1757 #endif
1758 		if (copied >= target) {
1759 			/* Do not sleep, just process backlog. */
1760 			release_sock(sk);
1761 			lock_sock(sk);
1762 		} else
1763 			sk_wait_data(sk, &timeo);
1764 
1765 #ifdef CONFIG_NET_DMA
1766 		tcp_service_net_dma(sk, false);  /* Don't block */
1767 		tp->ucopy.wakeup = 0;
1768 #endif
1769 
1770 		if (user_recv) {
1771 			int chunk;
1772 
1773 			/* __ Restore normal policy in scheduler __ */
1774 
1775 			if ((chunk = len - tp->ucopy.len) != 0) {
1776 				NET_ADD_STATS_USER(sock_net(sk), LINUX_MIB_TCPDIRECTCOPYFROMBACKLOG, chunk);
1777 				len -= chunk;
1778 				copied += chunk;
1779 			}
1780 
1781 			if (tp->rcv_nxt == tp->copied_seq &&
1782 			    !skb_queue_empty(&tp->ucopy.prequeue)) {
1783 do_prequeue:
1784 				tcp_prequeue_process(sk);
1785 
1786 				if ((chunk = len - tp->ucopy.len) != 0) {
1787 					NET_ADD_STATS_USER(sock_net(sk), LINUX_MIB_TCPDIRECTCOPYFROMPREQUEUE, chunk);
1788 					len -= chunk;
1789 					copied += chunk;
1790 				}
1791 			}
1792 		}
1793 		if ((flags & MSG_PEEK) &&
1794 		    (peek_seq - copied - urg_hole != tp->copied_seq)) {
1795 			net_dbg_ratelimited("TCP(%s:%d): Application bug, race in MSG_PEEK\n",
1796 					    current->comm,
1797 					    task_pid_nr(current));
1798 			peek_seq = tp->copied_seq;
1799 		}
1800 		continue;
1801 
1802 	found_ok_skb:
1803 		/* Ok so how much can we use? */
1804 		used = skb->len - offset;
1805 		if (len < used)
1806 			used = len;
1807 
1808 		/* Do we have urgent data here? */
1809 		if (tp->urg_data) {
1810 			u32 urg_offset = tp->urg_seq - *seq;
1811 			if (urg_offset < used) {
1812 				if (!urg_offset) {
1813 					if (!sock_flag(sk, SOCK_URGINLINE)) {
1814 						++*seq;
1815 						urg_hole++;
1816 						offset++;
1817 						used--;
1818 						if (!used)
1819 							goto skip_copy;
1820 					}
1821 				} else
1822 					used = urg_offset;
1823 			}
1824 		}
1825 
1826 		if (!(flags & MSG_TRUNC)) {
1827 #ifdef CONFIG_NET_DMA
1828 			if (!tp->ucopy.dma_chan && tp->ucopy.pinned_list)
1829 				tp->ucopy.dma_chan = net_dma_find_channel();
1830 
1831 			if (tp->ucopy.dma_chan) {
1832 				tp->ucopy.dma_cookie = dma_skb_copy_datagram_iovec(
1833 					tp->ucopy.dma_chan, skb, offset,
1834 					msg->msg_iov, used,
1835 					tp->ucopy.pinned_list);
1836 
1837 				if (tp->ucopy.dma_cookie < 0) {
1838 
1839 					pr_alert("%s: dma_cookie < 0\n",
1840 						 __func__);
1841 
1842 					/* Exception. Bailout! */
1843 					if (!copied)
1844 						copied = -EFAULT;
1845 					break;
1846 				}
1847 
1848 				dma_async_issue_pending(tp->ucopy.dma_chan);
1849 
1850 				if ((offset + used) == skb->len)
1851 					copied_early = true;
1852 
1853 			} else
1854 #endif
1855 			{
1856 				err = skb_copy_datagram_iovec(skb, offset,
1857 						msg->msg_iov, used);
1858 				if (err) {
1859 					/* Exception. Bailout! */
1860 					if (!copied)
1861 						copied = -EFAULT;
1862 					break;
1863 				}
1864 			}
1865 		}
1866 
1867 		*seq += used;
1868 		copied += used;
1869 		len -= used;
1870 
1871 		tcp_rcv_space_adjust(sk);
1872 
1873 skip_copy:
1874 		if (tp->urg_data && after(tp->copied_seq, tp->urg_seq)) {
1875 			tp->urg_data = 0;
1876 			tcp_fast_path_check(sk);
1877 		}
1878 		if (used + offset < skb->len)
1879 			continue;
1880 
1881 		if (tcp_hdr(skb)->fin)
1882 			goto found_fin_ok;
1883 		if (!(flags & MSG_PEEK)) {
1884 			sk_eat_skb(sk, skb, copied_early);
1885 			copied_early = false;
1886 		}
1887 		continue;
1888 
1889 	found_fin_ok:
1890 		/* Process the FIN. */
1891 		++*seq;
1892 		if (!(flags & MSG_PEEK)) {
1893 			sk_eat_skb(sk, skb, copied_early);
1894 			copied_early = false;
1895 		}
1896 		break;
1897 	} while (len > 0);
1898 
1899 	if (user_recv) {
1900 		if (!skb_queue_empty(&tp->ucopy.prequeue)) {
1901 			int chunk;
1902 
1903 			tp->ucopy.len = copied > 0 ? len : 0;
1904 
1905 			tcp_prequeue_process(sk);
1906 
1907 			if (copied > 0 && (chunk = len - tp->ucopy.len) != 0) {
1908 				NET_ADD_STATS_USER(sock_net(sk), LINUX_MIB_TCPDIRECTCOPYFROMPREQUEUE, chunk);
1909 				len -= chunk;
1910 				copied += chunk;
1911 			}
1912 		}
1913 
1914 		tp->ucopy.task = NULL;
1915 		tp->ucopy.len = 0;
1916 	}
1917 
1918 #ifdef CONFIG_NET_DMA
1919 	tcp_service_net_dma(sk, true);  /* Wait for queue to drain */
1920 	tp->ucopy.dma_chan = NULL;
1921 
1922 	if (tp->ucopy.pinned_list) {
1923 		dma_unpin_iovec_pages(tp->ucopy.pinned_list);
1924 		tp->ucopy.pinned_list = NULL;
1925 	}
1926 #endif
1927 
1928 	/* According to UNIX98, msg_name/msg_namelen are ignored
1929 	 * on connected socket. I was just happy when found this 8) --ANK
1930 	 */
1931 
1932 	/* Clean up data we have read: This will do ACK frames. */
1933 	tcp_cleanup_rbuf(sk, copied);
1934 
1935 	release_sock(sk);
1936 	return copied;
1937 
1938 out:
1939 	release_sock(sk);
1940 	return err;
1941 
1942 recv_urg:
1943 	err = tcp_recv_urg(sk, msg, len, flags);
1944 	goto out;
1945 
1946 recv_sndq:
1947 	err = tcp_peek_sndq(sk, msg, len);
1948 	goto out;
1949 }
1950 EXPORT_SYMBOL(tcp_recvmsg);
1951 
1952 void tcp_set_state(struct sock *sk, int state)
1953 {
1954 	int oldstate = sk->sk_state;
1955 
1956 	switch (state) {
1957 	case TCP_ESTABLISHED:
1958 		if (oldstate != TCP_ESTABLISHED)
1959 			TCP_INC_STATS(sock_net(sk), TCP_MIB_CURRESTAB);
1960 		break;
1961 
1962 	case TCP_CLOSE:
1963 		if (oldstate == TCP_CLOSE_WAIT || oldstate == TCP_ESTABLISHED)
1964 			TCP_INC_STATS(sock_net(sk), TCP_MIB_ESTABRESETS);
1965 
1966 		sk->sk_prot->unhash(sk);
1967 		if (inet_csk(sk)->icsk_bind_hash &&
1968 		    !(sk->sk_userlocks & SOCK_BINDPORT_LOCK))
1969 			inet_put_port(sk);
1970 		/* fall through */
1971 	default:
1972 		if (oldstate == TCP_ESTABLISHED)
1973 			TCP_DEC_STATS(sock_net(sk), TCP_MIB_CURRESTAB);
1974 	}
1975 
1976 	/* Change state AFTER socket is unhashed to avoid closed
1977 	 * socket sitting in hash tables.
1978 	 */
1979 	sk->sk_state = state;
1980 
1981 #ifdef STATE_TRACE
1982 	SOCK_DEBUG(sk, "TCP sk=%p, State %s -> %s\n", sk, statename[oldstate], statename[state]);
1983 #endif
1984 }
1985 EXPORT_SYMBOL_GPL(tcp_set_state);
1986 
1987 /*
1988  *	State processing on a close. This implements the state shift for
1989  *	sending our FIN frame. Note that we only send a FIN for some
1990  *	states. A shutdown() may have already sent the FIN, or we may be
1991  *	closed.
1992  */
1993 
1994 static const unsigned char new_state[16] = {
1995   /* current state:        new state:      action:	*/
1996   /* (Invalid)		*/ TCP_CLOSE,
1997   /* TCP_ESTABLISHED	*/ TCP_FIN_WAIT1 | TCP_ACTION_FIN,
1998   /* TCP_SYN_SENT	*/ TCP_CLOSE,
1999   /* TCP_SYN_RECV	*/ TCP_FIN_WAIT1 | TCP_ACTION_FIN,
2000   /* TCP_FIN_WAIT1	*/ TCP_FIN_WAIT1,
2001   /* TCP_FIN_WAIT2	*/ TCP_FIN_WAIT2,
2002   /* TCP_TIME_WAIT	*/ TCP_CLOSE,
2003   /* TCP_CLOSE		*/ TCP_CLOSE,
2004   /* TCP_CLOSE_WAIT	*/ TCP_LAST_ACK  | TCP_ACTION_FIN,
2005   /* TCP_LAST_ACK	*/ TCP_LAST_ACK,
2006   /* TCP_LISTEN		*/ TCP_CLOSE,
2007   /* TCP_CLOSING	*/ TCP_CLOSING,
2008 };
2009 
2010 static int tcp_close_state(struct sock *sk)
2011 {
2012 	int next = (int)new_state[sk->sk_state];
2013 	int ns = next & TCP_STATE_MASK;
2014 
2015 	tcp_set_state(sk, ns);
2016 
2017 	return next & TCP_ACTION_FIN;
2018 }
2019 
2020 /*
2021  *	Shutdown the sending side of a connection. Much like close except
2022  *	that we don't receive shut down or sock_set_flag(sk, SOCK_DEAD).
2023  */
2024 
2025 void tcp_shutdown(struct sock *sk, int how)
2026 {
2027 	/*	We need to grab some memory, and put together a FIN,
2028 	 *	and then put it into the queue to be sent.
2029 	 *		Tim MacKenzie(tym@dibbler.cs.monash.edu.au) 4 Dec '92.
2030 	 */
2031 	if (!(how & SEND_SHUTDOWN))
2032 		return;
2033 
2034 	/* If we've already sent a FIN, or it's a closed state, skip this. */
2035 	if ((1 << sk->sk_state) &
2036 	    (TCPF_ESTABLISHED | TCPF_SYN_SENT |
2037 	     TCPF_SYN_RECV | TCPF_CLOSE_WAIT)) {
2038 		/* Clear out any half completed packets.  FIN if needed. */
2039 		if (tcp_close_state(sk))
2040 			tcp_send_fin(sk);
2041 	}
2042 }
2043 EXPORT_SYMBOL(tcp_shutdown);
2044 
2045 bool tcp_check_oom(struct sock *sk, int shift)
2046 {
2047 	bool too_many_orphans, out_of_socket_memory;
2048 
2049 	too_many_orphans = tcp_too_many_orphans(sk, shift);
2050 	out_of_socket_memory = tcp_out_of_memory(sk);
2051 
2052 	if (too_many_orphans)
2053 		net_info_ratelimited("too many orphaned sockets\n");
2054 	if (out_of_socket_memory)
2055 		net_info_ratelimited("out of memory -- consider tuning tcp_mem\n");
2056 	return too_many_orphans || out_of_socket_memory;
2057 }
2058 
2059 void tcp_close(struct sock *sk, long timeout)
2060 {
2061 	struct sk_buff *skb;
2062 	int data_was_unread = 0;
2063 	int state;
2064 
2065 	lock_sock(sk);
2066 	sk->sk_shutdown = SHUTDOWN_MASK;
2067 
2068 	if (sk->sk_state == TCP_LISTEN) {
2069 		tcp_set_state(sk, TCP_CLOSE);
2070 
2071 		/* Special case. */
2072 		inet_csk_listen_stop(sk);
2073 
2074 		goto adjudge_to_death;
2075 	}
2076 
2077 	/*  We need to flush the recv. buffs.  We do this only on the
2078 	 *  descriptor close, not protocol-sourced closes, because the
2079 	 *  reader process may not have drained the data yet!
2080 	 */
2081 	while ((skb = __skb_dequeue(&sk->sk_receive_queue)) != NULL) {
2082 		u32 len = TCP_SKB_CB(skb)->end_seq - TCP_SKB_CB(skb)->seq -
2083 			  tcp_hdr(skb)->fin;
2084 		data_was_unread += len;
2085 		__kfree_skb(skb);
2086 	}
2087 
2088 	sk_mem_reclaim(sk);
2089 
2090 	/* If socket has been already reset (e.g. in tcp_reset()) - kill it. */
2091 	if (sk->sk_state == TCP_CLOSE)
2092 		goto adjudge_to_death;
2093 
2094 	/* As outlined in RFC 2525, section 2.17, we send a RST here because
2095 	 * data was lost. To witness the awful effects of the old behavior of
2096 	 * always doing a FIN, run an older 2.1.x kernel or 2.0.x, start a bulk
2097 	 * GET in an FTP client, suspend the process, wait for the client to
2098 	 * advertise a zero window, then kill -9 the FTP client, wheee...
2099 	 * Note: timeout is always zero in such a case.
2100 	 */
2101 	if (unlikely(tcp_sk(sk)->repair)) {
2102 		sk->sk_prot->disconnect(sk, 0);
2103 	} else if (data_was_unread) {
2104 		/* Unread data was tossed, zap the connection. */
2105 		NET_INC_STATS_USER(sock_net(sk), LINUX_MIB_TCPABORTONCLOSE);
2106 		tcp_set_state(sk, TCP_CLOSE);
2107 		tcp_send_active_reset(sk, sk->sk_allocation);
2108 	} else if (sock_flag(sk, SOCK_LINGER) && !sk->sk_lingertime) {
2109 		/* Check zero linger _after_ checking for unread data. */
2110 		sk->sk_prot->disconnect(sk, 0);
2111 		NET_INC_STATS_USER(sock_net(sk), LINUX_MIB_TCPABORTONDATA);
2112 	} else if (tcp_close_state(sk)) {
2113 		/* We FIN if the application ate all the data before
2114 		 * zapping the connection.
2115 		 */
2116 
2117 		/* RED-PEN. Formally speaking, we have broken TCP state
2118 		 * machine. State transitions:
2119 		 *
2120 		 * TCP_ESTABLISHED -> TCP_FIN_WAIT1
2121 		 * TCP_SYN_RECV	-> TCP_FIN_WAIT1 (forget it, it's impossible)
2122 		 * TCP_CLOSE_WAIT -> TCP_LAST_ACK
2123 		 *
2124 		 * are legal only when FIN has been sent (i.e. in window),
2125 		 * rather than queued out of window. Purists blame.
2126 		 *
2127 		 * F.e. "RFC state" is ESTABLISHED,
2128 		 * if Linux state is FIN-WAIT-1, but FIN is still not sent.
2129 		 *
2130 		 * The visible declinations are that sometimes
2131 		 * we enter time-wait state, when it is not required really
2132 		 * (harmless), do not send active resets, when they are
2133 		 * required by specs (TCP_ESTABLISHED, TCP_CLOSE_WAIT, when
2134 		 * they look as CLOSING or LAST_ACK for Linux)
2135 		 * Probably, I missed some more holelets.
2136 		 * 						--ANK
2137 		 * XXX (TFO) - To start off we don't support SYN+ACK+FIN
2138 		 * in a single packet! (May consider it later but will
2139 		 * probably need API support or TCP_CORK SYN-ACK until
2140 		 * data is written and socket is closed.)
2141 		 */
2142 		tcp_send_fin(sk);
2143 	}
2144 
2145 	sk_stream_wait_close(sk, timeout);
2146 
2147 adjudge_to_death:
2148 	state = sk->sk_state;
2149 	sock_hold(sk);
2150 	sock_orphan(sk);
2151 
2152 	/* It is the last release_sock in its life. It will remove backlog. */
2153 	release_sock(sk);
2154 
2155 
2156 	/* Now socket is owned by kernel and we acquire BH lock
2157 	   to finish close. No need to check for user refs.
2158 	 */
2159 	local_bh_disable();
2160 	bh_lock_sock(sk);
2161 	WARN_ON(sock_owned_by_user(sk));
2162 
2163 	percpu_counter_inc(sk->sk_prot->orphan_count);
2164 
2165 	/* Have we already been destroyed by a softirq or backlog? */
2166 	if (state != TCP_CLOSE && sk->sk_state == TCP_CLOSE)
2167 		goto out;
2168 
2169 	/*	This is a (useful) BSD violating of the RFC. There is a
2170 	 *	problem with TCP as specified in that the other end could
2171 	 *	keep a socket open forever with no application left this end.
2172 	 *	We use a 3 minute timeout (about the same as BSD) then kill
2173 	 *	our end. If they send after that then tough - BUT: long enough
2174 	 *	that we won't make the old 4*rto = almost no time - whoops
2175 	 *	reset mistake.
2176 	 *
2177 	 *	Nope, it was not mistake. It is really desired behaviour
2178 	 *	f.e. on http servers, when such sockets are useless, but
2179 	 *	consume significant resources. Let's do it with special
2180 	 *	linger2	option.					--ANK
2181 	 */
2182 
2183 	if (sk->sk_state == TCP_FIN_WAIT2) {
2184 		struct tcp_sock *tp = tcp_sk(sk);
2185 		if (tp->linger2 < 0) {
2186 			tcp_set_state(sk, TCP_CLOSE);
2187 			tcp_send_active_reset(sk, GFP_ATOMIC);
2188 			NET_INC_STATS_BH(sock_net(sk),
2189 					LINUX_MIB_TCPABORTONLINGER);
2190 		} else {
2191 			const int tmo = tcp_fin_time(sk);
2192 
2193 			if (tmo > TCP_TIMEWAIT_LEN) {
2194 				inet_csk_reset_keepalive_timer(sk,
2195 						tmo - TCP_TIMEWAIT_LEN);
2196 			} else {
2197 				tcp_time_wait(sk, TCP_FIN_WAIT2, tmo);
2198 				goto out;
2199 			}
2200 		}
2201 	}
2202 	if (sk->sk_state != TCP_CLOSE) {
2203 		sk_mem_reclaim(sk);
2204 		if (tcp_check_oom(sk, 0)) {
2205 			tcp_set_state(sk, TCP_CLOSE);
2206 			tcp_send_active_reset(sk, GFP_ATOMIC);
2207 			NET_INC_STATS_BH(sock_net(sk),
2208 					LINUX_MIB_TCPABORTONMEMORY);
2209 		}
2210 	}
2211 
2212 	if (sk->sk_state == TCP_CLOSE) {
2213 		struct request_sock *req = tcp_sk(sk)->fastopen_rsk;
2214 		/* We could get here with a non-NULL req if the socket is
2215 		 * aborted (e.g., closed with unread data) before 3WHS
2216 		 * finishes.
2217 		 */
2218 		if (req != NULL)
2219 			reqsk_fastopen_remove(sk, req, false);
2220 		inet_csk_destroy_sock(sk);
2221 	}
2222 	/* Otherwise, socket is reprieved until protocol close. */
2223 
2224 out:
2225 	bh_unlock_sock(sk);
2226 	local_bh_enable();
2227 	sock_put(sk);
2228 }
2229 EXPORT_SYMBOL(tcp_close);
2230 
2231 /* These states need RST on ABORT according to RFC793 */
2232 
2233 static inline bool tcp_need_reset(int state)
2234 {
2235 	return (1 << state) &
2236 	       (TCPF_ESTABLISHED | TCPF_CLOSE_WAIT | TCPF_FIN_WAIT1 |
2237 		TCPF_FIN_WAIT2 | TCPF_SYN_RECV);
2238 }
2239 
2240 int tcp_disconnect(struct sock *sk, int flags)
2241 {
2242 	struct inet_sock *inet = inet_sk(sk);
2243 	struct inet_connection_sock *icsk = inet_csk(sk);
2244 	struct tcp_sock *tp = tcp_sk(sk);
2245 	int err = 0;
2246 	int old_state = sk->sk_state;
2247 
2248 	if (old_state != TCP_CLOSE)
2249 		tcp_set_state(sk, TCP_CLOSE);
2250 
2251 	/* ABORT function of RFC793 */
2252 	if (old_state == TCP_LISTEN) {
2253 		inet_csk_listen_stop(sk);
2254 	} else if (unlikely(tp->repair)) {
2255 		sk->sk_err = ECONNABORTED;
2256 	} else if (tcp_need_reset(old_state) ||
2257 		   (tp->snd_nxt != tp->write_seq &&
2258 		    (1 << old_state) & (TCPF_CLOSING | TCPF_LAST_ACK))) {
2259 		/* The last check adjusts for discrepancy of Linux wrt. RFC
2260 		 * states
2261 		 */
2262 		tcp_send_active_reset(sk, gfp_any());
2263 		sk->sk_err = ECONNRESET;
2264 	} else if (old_state == TCP_SYN_SENT)
2265 		sk->sk_err = ECONNRESET;
2266 
2267 	tcp_clear_xmit_timers(sk);
2268 	__skb_queue_purge(&sk->sk_receive_queue);
2269 	tcp_write_queue_purge(sk);
2270 	__skb_queue_purge(&tp->out_of_order_queue);
2271 #ifdef CONFIG_NET_DMA
2272 	__skb_queue_purge(&sk->sk_async_wait_queue);
2273 #endif
2274 
2275 	inet->inet_dport = 0;
2276 
2277 	if (!(sk->sk_userlocks & SOCK_BINDADDR_LOCK))
2278 		inet_reset_saddr(sk);
2279 
2280 	sk->sk_shutdown = 0;
2281 	sock_reset_flag(sk, SOCK_DONE);
2282 	tp->srtt = 0;
2283 	if ((tp->write_seq += tp->max_window + 2) == 0)
2284 		tp->write_seq = 1;
2285 	icsk->icsk_backoff = 0;
2286 	tp->snd_cwnd = 2;
2287 	icsk->icsk_probes_out = 0;
2288 	tp->packets_out = 0;
2289 	tp->snd_ssthresh = TCP_INFINITE_SSTHRESH;
2290 	tp->snd_cwnd_cnt = 0;
2291 	tp->window_clamp = 0;
2292 	tcp_set_ca_state(sk, TCP_CA_Open);
2293 	tcp_clear_retrans(tp);
2294 	inet_csk_delack_init(sk);
2295 	tcp_init_send_head(sk);
2296 	memset(&tp->rx_opt, 0, sizeof(tp->rx_opt));
2297 	__sk_dst_reset(sk);
2298 
2299 	WARN_ON(inet->inet_num && !icsk->icsk_bind_hash);
2300 
2301 	sk->sk_error_report(sk);
2302 	return err;
2303 }
2304 EXPORT_SYMBOL(tcp_disconnect);
2305 
2306 void tcp_sock_destruct(struct sock *sk)
2307 {
2308 	inet_sock_destruct(sk);
2309 
2310 	kfree(inet_csk(sk)->icsk_accept_queue.fastopenq);
2311 }
2312 
2313 static inline bool tcp_can_repair_sock(const struct sock *sk)
2314 {
2315 	return ns_capable(sock_net(sk)->user_ns, CAP_NET_ADMIN) &&
2316 		((1 << sk->sk_state) & (TCPF_CLOSE | TCPF_ESTABLISHED));
2317 }
2318 
2319 static int tcp_repair_options_est(struct tcp_sock *tp,
2320 		struct tcp_repair_opt __user *optbuf, unsigned int len)
2321 {
2322 	struct tcp_repair_opt opt;
2323 
2324 	while (len >= sizeof(opt)) {
2325 		if (copy_from_user(&opt, optbuf, sizeof(opt)))
2326 			return -EFAULT;
2327 
2328 		optbuf++;
2329 		len -= sizeof(opt);
2330 
2331 		switch (opt.opt_code) {
2332 		case TCPOPT_MSS:
2333 			tp->rx_opt.mss_clamp = opt.opt_val;
2334 			break;
2335 		case TCPOPT_WINDOW:
2336 			{
2337 				u16 snd_wscale = opt.opt_val & 0xFFFF;
2338 				u16 rcv_wscale = opt.opt_val >> 16;
2339 
2340 				if (snd_wscale > 14 || rcv_wscale > 14)
2341 					return -EFBIG;
2342 
2343 				tp->rx_opt.snd_wscale = snd_wscale;
2344 				tp->rx_opt.rcv_wscale = rcv_wscale;
2345 				tp->rx_opt.wscale_ok = 1;
2346 			}
2347 			break;
2348 		case TCPOPT_SACK_PERM:
2349 			if (opt.opt_val != 0)
2350 				return -EINVAL;
2351 
2352 			tp->rx_opt.sack_ok |= TCP_SACK_SEEN;
2353 			if (sysctl_tcp_fack)
2354 				tcp_enable_fack(tp);
2355 			break;
2356 		case TCPOPT_TIMESTAMP:
2357 			if (opt.opt_val != 0)
2358 				return -EINVAL;
2359 
2360 			tp->rx_opt.tstamp_ok = 1;
2361 			break;
2362 		}
2363 	}
2364 
2365 	return 0;
2366 }
2367 
2368 /*
2369  *	Socket option code for TCP.
2370  */
2371 static int do_tcp_setsockopt(struct sock *sk, int level,
2372 		int optname, char __user *optval, unsigned int optlen)
2373 {
2374 	struct tcp_sock *tp = tcp_sk(sk);
2375 	struct inet_connection_sock *icsk = inet_csk(sk);
2376 	int val;
2377 	int err = 0;
2378 
2379 	/* These are data/string values, all the others are ints */
2380 	switch (optname) {
2381 	case TCP_CONGESTION: {
2382 		char name[TCP_CA_NAME_MAX];
2383 
2384 		if (optlen < 1)
2385 			return -EINVAL;
2386 
2387 		val = strncpy_from_user(name, optval,
2388 					min_t(long, TCP_CA_NAME_MAX-1, optlen));
2389 		if (val < 0)
2390 			return -EFAULT;
2391 		name[val] = 0;
2392 
2393 		lock_sock(sk);
2394 		err = tcp_set_congestion_control(sk, name);
2395 		release_sock(sk);
2396 		return err;
2397 	}
2398 	default:
2399 		/* fallthru */
2400 		break;
2401 	}
2402 
2403 	if (optlen < sizeof(int))
2404 		return -EINVAL;
2405 
2406 	if (get_user(val, (int __user *)optval))
2407 		return -EFAULT;
2408 
2409 	lock_sock(sk);
2410 
2411 	switch (optname) {
2412 	case TCP_MAXSEG:
2413 		/* Values greater than interface MTU won't take effect. However
2414 		 * at the point when this call is done we typically don't yet
2415 		 * know which interface is going to be used */
2416 		if (val < TCP_MIN_MSS || val > MAX_TCP_WINDOW) {
2417 			err = -EINVAL;
2418 			break;
2419 		}
2420 		tp->rx_opt.user_mss = val;
2421 		break;
2422 
2423 	case TCP_NODELAY:
2424 		if (val) {
2425 			/* TCP_NODELAY is weaker than TCP_CORK, so that
2426 			 * this option on corked socket is remembered, but
2427 			 * it is not activated until cork is cleared.
2428 			 *
2429 			 * However, when TCP_NODELAY is set we make
2430 			 * an explicit push, which overrides even TCP_CORK
2431 			 * for currently queued segments.
2432 			 */
2433 			tp->nonagle |= TCP_NAGLE_OFF|TCP_NAGLE_PUSH;
2434 			tcp_push_pending_frames(sk);
2435 		} else {
2436 			tp->nonagle &= ~TCP_NAGLE_OFF;
2437 		}
2438 		break;
2439 
2440 	case TCP_THIN_LINEAR_TIMEOUTS:
2441 		if (val < 0 || val > 1)
2442 			err = -EINVAL;
2443 		else
2444 			tp->thin_lto = val;
2445 		break;
2446 
2447 	case TCP_THIN_DUPACK:
2448 		if (val < 0 || val > 1)
2449 			err = -EINVAL;
2450 		else
2451 			tp->thin_dupack = val;
2452 			if (tp->thin_dupack)
2453 				tcp_disable_early_retrans(tp);
2454 		break;
2455 
2456 	case TCP_REPAIR:
2457 		if (!tcp_can_repair_sock(sk))
2458 			err = -EPERM;
2459 		else if (val == 1) {
2460 			tp->repair = 1;
2461 			sk->sk_reuse = SK_FORCE_REUSE;
2462 			tp->repair_queue = TCP_NO_QUEUE;
2463 		} else if (val == 0) {
2464 			tp->repair = 0;
2465 			sk->sk_reuse = SK_NO_REUSE;
2466 			tcp_send_window_probe(sk);
2467 		} else
2468 			err = -EINVAL;
2469 
2470 		break;
2471 
2472 	case TCP_REPAIR_QUEUE:
2473 		if (!tp->repair)
2474 			err = -EPERM;
2475 		else if (val < TCP_QUEUES_NR)
2476 			tp->repair_queue = val;
2477 		else
2478 			err = -EINVAL;
2479 		break;
2480 
2481 	case TCP_QUEUE_SEQ:
2482 		if (sk->sk_state != TCP_CLOSE)
2483 			err = -EPERM;
2484 		else if (tp->repair_queue == TCP_SEND_QUEUE)
2485 			tp->write_seq = val;
2486 		else if (tp->repair_queue == TCP_RECV_QUEUE)
2487 			tp->rcv_nxt = val;
2488 		else
2489 			err = -EINVAL;
2490 		break;
2491 
2492 	case TCP_REPAIR_OPTIONS:
2493 		if (!tp->repair)
2494 			err = -EINVAL;
2495 		else if (sk->sk_state == TCP_ESTABLISHED)
2496 			err = tcp_repair_options_est(tp,
2497 					(struct tcp_repair_opt __user *)optval,
2498 					optlen);
2499 		else
2500 			err = -EPERM;
2501 		break;
2502 
2503 	case TCP_CORK:
2504 		/* When set indicates to always queue non-full frames.
2505 		 * Later the user clears this option and we transmit
2506 		 * any pending partial frames in the queue.  This is
2507 		 * meant to be used alongside sendfile() to get properly
2508 		 * filled frames when the user (for example) must write
2509 		 * out headers with a write() call first and then use
2510 		 * sendfile to send out the data parts.
2511 		 *
2512 		 * TCP_CORK can be set together with TCP_NODELAY and it is
2513 		 * stronger than TCP_NODELAY.
2514 		 */
2515 		if (val) {
2516 			tp->nonagle |= TCP_NAGLE_CORK;
2517 		} else {
2518 			tp->nonagle &= ~TCP_NAGLE_CORK;
2519 			if (tp->nonagle&TCP_NAGLE_OFF)
2520 				tp->nonagle |= TCP_NAGLE_PUSH;
2521 			tcp_push_pending_frames(sk);
2522 		}
2523 		break;
2524 
2525 	case TCP_KEEPIDLE:
2526 		if (val < 1 || val > MAX_TCP_KEEPIDLE)
2527 			err = -EINVAL;
2528 		else {
2529 			tp->keepalive_time = val * HZ;
2530 			if (sock_flag(sk, SOCK_KEEPOPEN) &&
2531 			    !((1 << sk->sk_state) &
2532 			      (TCPF_CLOSE | TCPF_LISTEN))) {
2533 				u32 elapsed = keepalive_time_elapsed(tp);
2534 				if (tp->keepalive_time > elapsed)
2535 					elapsed = tp->keepalive_time - elapsed;
2536 				else
2537 					elapsed = 0;
2538 				inet_csk_reset_keepalive_timer(sk, elapsed);
2539 			}
2540 		}
2541 		break;
2542 	case TCP_KEEPINTVL:
2543 		if (val < 1 || val > MAX_TCP_KEEPINTVL)
2544 			err = -EINVAL;
2545 		else
2546 			tp->keepalive_intvl = val * HZ;
2547 		break;
2548 	case TCP_KEEPCNT:
2549 		if (val < 1 || val > MAX_TCP_KEEPCNT)
2550 			err = -EINVAL;
2551 		else
2552 			tp->keepalive_probes = val;
2553 		break;
2554 	case TCP_SYNCNT:
2555 		if (val < 1 || val > MAX_TCP_SYNCNT)
2556 			err = -EINVAL;
2557 		else
2558 			icsk->icsk_syn_retries = val;
2559 		break;
2560 
2561 	case TCP_LINGER2:
2562 		if (val < 0)
2563 			tp->linger2 = -1;
2564 		else if (val > sysctl_tcp_fin_timeout / HZ)
2565 			tp->linger2 = 0;
2566 		else
2567 			tp->linger2 = val * HZ;
2568 		break;
2569 
2570 	case TCP_DEFER_ACCEPT:
2571 		/* Translate value in seconds to number of retransmits */
2572 		icsk->icsk_accept_queue.rskq_defer_accept =
2573 			secs_to_retrans(val, TCP_TIMEOUT_INIT / HZ,
2574 					TCP_RTO_MAX / HZ);
2575 		break;
2576 
2577 	case TCP_WINDOW_CLAMP:
2578 		if (!val) {
2579 			if (sk->sk_state != TCP_CLOSE) {
2580 				err = -EINVAL;
2581 				break;
2582 			}
2583 			tp->window_clamp = 0;
2584 		} else
2585 			tp->window_clamp = val < SOCK_MIN_RCVBUF / 2 ?
2586 						SOCK_MIN_RCVBUF / 2 : val;
2587 		break;
2588 
2589 	case TCP_QUICKACK:
2590 		if (!val) {
2591 			icsk->icsk_ack.pingpong = 1;
2592 		} else {
2593 			icsk->icsk_ack.pingpong = 0;
2594 			if ((1 << sk->sk_state) &
2595 			    (TCPF_ESTABLISHED | TCPF_CLOSE_WAIT) &&
2596 			    inet_csk_ack_scheduled(sk)) {
2597 				icsk->icsk_ack.pending |= ICSK_ACK_PUSHED;
2598 				tcp_cleanup_rbuf(sk, 1);
2599 				if (!(val & 1))
2600 					icsk->icsk_ack.pingpong = 1;
2601 			}
2602 		}
2603 		break;
2604 
2605 #ifdef CONFIG_TCP_MD5SIG
2606 	case TCP_MD5SIG:
2607 		/* Read the IP->Key mappings from userspace */
2608 		err = tp->af_specific->md5_parse(sk, optval, optlen);
2609 		break;
2610 #endif
2611 	case TCP_USER_TIMEOUT:
2612 		/* Cap the max timeout in ms TCP will retry/retrans
2613 		 * before giving up and aborting (ETIMEDOUT) a connection.
2614 		 */
2615 		if (val < 0)
2616 			err = -EINVAL;
2617 		else
2618 			icsk->icsk_user_timeout = msecs_to_jiffies(val);
2619 		break;
2620 
2621 	case TCP_FASTOPEN:
2622 		if (val >= 0 && ((1 << sk->sk_state) & (TCPF_CLOSE |
2623 		    TCPF_LISTEN)))
2624 			err = fastopen_init_queue(sk, val);
2625 		else
2626 			err = -EINVAL;
2627 		break;
2628 	case TCP_TIMESTAMP:
2629 		if (!tp->repair)
2630 			err = -EPERM;
2631 		else
2632 			tp->tsoffset = val - tcp_time_stamp;
2633 		break;
2634 	default:
2635 		err = -ENOPROTOOPT;
2636 		break;
2637 	}
2638 
2639 	release_sock(sk);
2640 	return err;
2641 }
2642 
2643 int tcp_setsockopt(struct sock *sk, int level, int optname, char __user *optval,
2644 		   unsigned int optlen)
2645 {
2646 	const struct inet_connection_sock *icsk = inet_csk(sk);
2647 
2648 	if (level != SOL_TCP)
2649 		return icsk->icsk_af_ops->setsockopt(sk, level, optname,
2650 						     optval, optlen);
2651 	return do_tcp_setsockopt(sk, level, optname, optval, optlen);
2652 }
2653 EXPORT_SYMBOL(tcp_setsockopt);
2654 
2655 #ifdef CONFIG_COMPAT
2656 int compat_tcp_setsockopt(struct sock *sk, int level, int optname,
2657 			  char __user *optval, unsigned int optlen)
2658 {
2659 	if (level != SOL_TCP)
2660 		return inet_csk_compat_setsockopt(sk, level, optname,
2661 						  optval, optlen);
2662 	return do_tcp_setsockopt(sk, level, optname, optval, optlen);
2663 }
2664 EXPORT_SYMBOL(compat_tcp_setsockopt);
2665 #endif
2666 
2667 /* Return information about state of tcp endpoint in API format. */
2668 void tcp_get_info(const struct sock *sk, struct tcp_info *info)
2669 {
2670 	const struct tcp_sock *tp = tcp_sk(sk);
2671 	const struct inet_connection_sock *icsk = inet_csk(sk);
2672 	u32 now = tcp_time_stamp;
2673 
2674 	memset(info, 0, sizeof(*info));
2675 
2676 	info->tcpi_state = sk->sk_state;
2677 	info->tcpi_ca_state = icsk->icsk_ca_state;
2678 	info->tcpi_retransmits = icsk->icsk_retransmits;
2679 	info->tcpi_probes = icsk->icsk_probes_out;
2680 	info->tcpi_backoff = icsk->icsk_backoff;
2681 
2682 	if (tp->rx_opt.tstamp_ok)
2683 		info->tcpi_options |= TCPI_OPT_TIMESTAMPS;
2684 	if (tcp_is_sack(tp))
2685 		info->tcpi_options |= TCPI_OPT_SACK;
2686 	if (tp->rx_opt.wscale_ok) {
2687 		info->tcpi_options |= TCPI_OPT_WSCALE;
2688 		info->tcpi_snd_wscale = tp->rx_opt.snd_wscale;
2689 		info->tcpi_rcv_wscale = tp->rx_opt.rcv_wscale;
2690 	}
2691 
2692 	if (tp->ecn_flags & TCP_ECN_OK)
2693 		info->tcpi_options |= TCPI_OPT_ECN;
2694 	if (tp->ecn_flags & TCP_ECN_SEEN)
2695 		info->tcpi_options |= TCPI_OPT_ECN_SEEN;
2696 	if (tp->syn_data_acked)
2697 		info->tcpi_options |= TCPI_OPT_SYN_DATA;
2698 
2699 	info->tcpi_rto = jiffies_to_usecs(icsk->icsk_rto);
2700 	info->tcpi_ato = jiffies_to_usecs(icsk->icsk_ack.ato);
2701 	info->tcpi_snd_mss = tp->mss_cache;
2702 	info->tcpi_rcv_mss = icsk->icsk_ack.rcv_mss;
2703 
2704 	if (sk->sk_state == TCP_LISTEN) {
2705 		info->tcpi_unacked = sk->sk_ack_backlog;
2706 		info->tcpi_sacked = sk->sk_max_ack_backlog;
2707 	} else {
2708 		info->tcpi_unacked = tp->packets_out;
2709 		info->tcpi_sacked = tp->sacked_out;
2710 	}
2711 	info->tcpi_lost = tp->lost_out;
2712 	info->tcpi_retrans = tp->retrans_out;
2713 	info->tcpi_fackets = tp->fackets_out;
2714 
2715 	info->tcpi_last_data_sent = jiffies_to_msecs(now - tp->lsndtime);
2716 	info->tcpi_last_data_recv = jiffies_to_msecs(now - icsk->icsk_ack.lrcvtime);
2717 	info->tcpi_last_ack_recv = jiffies_to_msecs(now - tp->rcv_tstamp);
2718 
2719 	info->tcpi_pmtu = icsk->icsk_pmtu_cookie;
2720 	info->tcpi_rcv_ssthresh = tp->rcv_ssthresh;
2721 	info->tcpi_rtt = jiffies_to_usecs(tp->srtt)>>3;
2722 	info->tcpi_rttvar = jiffies_to_usecs(tp->mdev)>>2;
2723 	info->tcpi_snd_ssthresh = tp->snd_ssthresh;
2724 	info->tcpi_snd_cwnd = tp->snd_cwnd;
2725 	info->tcpi_advmss = tp->advmss;
2726 	info->tcpi_reordering = tp->reordering;
2727 
2728 	info->tcpi_rcv_rtt = jiffies_to_usecs(tp->rcv_rtt_est.rtt)>>3;
2729 	info->tcpi_rcv_space = tp->rcvq_space.space;
2730 
2731 	info->tcpi_total_retrans = tp->total_retrans;
2732 }
2733 EXPORT_SYMBOL_GPL(tcp_get_info);
2734 
2735 static int do_tcp_getsockopt(struct sock *sk, int level,
2736 		int optname, char __user *optval, int __user *optlen)
2737 {
2738 	struct inet_connection_sock *icsk = inet_csk(sk);
2739 	struct tcp_sock *tp = tcp_sk(sk);
2740 	int val, len;
2741 
2742 	if (get_user(len, optlen))
2743 		return -EFAULT;
2744 
2745 	len = min_t(unsigned int, len, sizeof(int));
2746 
2747 	if (len < 0)
2748 		return -EINVAL;
2749 
2750 	switch (optname) {
2751 	case TCP_MAXSEG:
2752 		val = tp->mss_cache;
2753 		if (!val && ((1 << sk->sk_state) & (TCPF_CLOSE | TCPF_LISTEN)))
2754 			val = tp->rx_opt.user_mss;
2755 		if (tp->repair)
2756 			val = tp->rx_opt.mss_clamp;
2757 		break;
2758 	case TCP_NODELAY:
2759 		val = !!(tp->nonagle&TCP_NAGLE_OFF);
2760 		break;
2761 	case TCP_CORK:
2762 		val = !!(tp->nonagle&TCP_NAGLE_CORK);
2763 		break;
2764 	case TCP_KEEPIDLE:
2765 		val = keepalive_time_when(tp) / HZ;
2766 		break;
2767 	case TCP_KEEPINTVL:
2768 		val = keepalive_intvl_when(tp) / HZ;
2769 		break;
2770 	case TCP_KEEPCNT:
2771 		val = keepalive_probes(tp);
2772 		break;
2773 	case TCP_SYNCNT:
2774 		val = icsk->icsk_syn_retries ? : sysctl_tcp_syn_retries;
2775 		break;
2776 	case TCP_LINGER2:
2777 		val = tp->linger2;
2778 		if (val >= 0)
2779 			val = (val ? : sysctl_tcp_fin_timeout) / HZ;
2780 		break;
2781 	case TCP_DEFER_ACCEPT:
2782 		val = retrans_to_secs(icsk->icsk_accept_queue.rskq_defer_accept,
2783 				      TCP_TIMEOUT_INIT / HZ, TCP_RTO_MAX / HZ);
2784 		break;
2785 	case TCP_WINDOW_CLAMP:
2786 		val = tp->window_clamp;
2787 		break;
2788 	case TCP_INFO: {
2789 		struct tcp_info info;
2790 
2791 		if (get_user(len, optlen))
2792 			return -EFAULT;
2793 
2794 		tcp_get_info(sk, &info);
2795 
2796 		len = min_t(unsigned int, len, sizeof(info));
2797 		if (put_user(len, optlen))
2798 			return -EFAULT;
2799 		if (copy_to_user(optval, &info, len))
2800 			return -EFAULT;
2801 		return 0;
2802 	}
2803 	case TCP_QUICKACK:
2804 		val = !icsk->icsk_ack.pingpong;
2805 		break;
2806 
2807 	case TCP_CONGESTION:
2808 		if (get_user(len, optlen))
2809 			return -EFAULT;
2810 		len = min_t(unsigned int, len, TCP_CA_NAME_MAX);
2811 		if (put_user(len, optlen))
2812 			return -EFAULT;
2813 		if (copy_to_user(optval, icsk->icsk_ca_ops->name, len))
2814 			return -EFAULT;
2815 		return 0;
2816 
2817 	case TCP_THIN_LINEAR_TIMEOUTS:
2818 		val = tp->thin_lto;
2819 		break;
2820 	case TCP_THIN_DUPACK:
2821 		val = tp->thin_dupack;
2822 		break;
2823 
2824 	case TCP_REPAIR:
2825 		val = tp->repair;
2826 		break;
2827 
2828 	case TCP_REPAIR_QUEUE:
2829 		if (tp->repair)
2830 			val = tp->repair_queue;
2831 		else
2832 			return -EINVAL;
2833 		break;
2834 
2835 	case TCP_QUEUE_SEQ:
2836 		if (tp->repair_queue == TCP_SEND_QUEUE)
2837 			val = tp->write_seq;
2838 		else if (tp->repair_queue == TCP_RECV_QUEUE)
2839 			val = tp->rcv_nxt;
2840 		else
2841 			return -EINVAL;
2842 		break;
2843 
2844 	case TCP_USER_TIMEOUT:
2845 		val = jiffies_to_msecs(icsk->icsk_user_timeout);
2846 		break;
2847 	case TCP_TIMESTAMP:
2848 		val = tcp_time_stamp + tp->tsoffset;
2849 		break;
2850 	default:
2851 		return -ENOPROTOOPT;
2852 	}
2853 
2854 	if (put_user(len, optlen))
2855 		return -EFAULT;
2856 	if (copy_to_user(optval, &val, len))
2857 		return -EFAULT;
2858 	return 0;
2859 }
2860 
2861 int tcp_getsockopt(struct sock *sk, int level, int optname, char __user *optval,
2862 		   int __user *optlen)
2863 {
2864 	struct inet_connection_sock *icsk = inet_csk(sk);
2865 
2866 	if (level != SOL_TCP)
2867 		return icsk->icsk_af_ops->getsockopt(sk, level, optname,
2868 						     optval, optlen);
2869 	return do_tcp_getsockopt(sk, level, optname, optval, optlen);
2870 }
2871 EXPORT_SYMBOL(tcp_getsockopt);
2872 
2873 #ifdef CONFIG_COMPAT
2874 int compat_tcp_getsockopt(struct sock *sk, int level, int optname,
2875 			  char __user *optval, int __user *optlen)
2876 {
2877 	if (level != SOL_TCP)
2878 		return inet_csk_compat_getsockopt(sk, level, optname,
2879 						  optval, optlen);
2880 	return do_tcp_getsockopt(sk, level, optname, optval, optlen);
2881 }
2882 EXPORT_SYMBOL(compat_tcp_getsockopt);
2883 #endif
2884 
2885 #ifdef CONFIG_TCP_MD5SIG
2886 static struct tcp_md5sig_pool __percpu *tcp_md5sig_pool __read_mostly;
2887 static DEFINE_MUTEX(tcp_md5sig_mutex);
2888 
2889 static void __tcp_free_md5sig_pool(struct tcp_md5sig_pool __percpu *pool)
2890 {
2891 	int cpu;
2892 
2893 	for_each_possible_cpu(cpu) {
2894 		struct tcp_md5sig_pool *p = per_cpu_ptr(pool, cpu);
2895 
2896 		if (p->md5_desc.tfm)
2897 			crypto_free_hash(p->md5_desc.tfm);
2898 	}
2899 	free_percpu(pool);
2900 }
2901 
2902 static void __tcp_alloc_md5sig_pool(void)
2903 {
2904 	int cpu;
2905 	struct tcp_md5sig_pool __percpu *pool;
2906 
2907 	pool = alloc_percpu(struct tcp_md5sig_pool);
2908 	if (!pool)
2909 		return;
2910 
2911 	for_each_possible_cpu(cpu) {
2912 		struct crypto_hash *hash;
2913 
2914 		hash = crypto_alloc_hash("md5", 0, CRYPTO_ALG_ASYNC);
2915 		if (IS_ERR_OR_NULL(hash))
2916 			goto out_free;
2917 
2918 		per_cpu_ptr(pool, cpu)->md5_desc.tfm = hash;
2919 	}
2920 	/* before setting tcp_md5sig_pool, we must commit all writes
2921 	 * to memory. See ACCESS_ONCE() in tcp_get_md5sig_pool()
2922 	 */
2923 	smp_wmb();
2924 	tcp_md5sig_pool = pool;
2925 	return;
2926 out_free:
2927 	__tcp_free_md5sig_pool(pool);
2928 }
2929 
2930 bool tcp_alloc_md5sig_pool(void)
2931 {
2932 	if (unlikely(!tcp_md5sig_pool)) {
2933 		mutex_lock(&tcp_md5sig_mutex);
2934 
2935 		if (!tcp_md5sig_pool)
2936 			__tcp_alloc_md5sig_pool();
2937 
2938 		mutex_unlock(&tcp_md5sig_mutex);
2939 	}
2940 	return tcp_md5sig_pool != NULL;
2941 }
2942 EXPORT_SYMBOL(tcp_alloc_md5sig_pool);
2943 
2944 
2945 /**
2946  *	tcp_get_md5sig_pool - get md5sig_pool for this user
2947  *
2948  *	We use percpu structure, so if we succeed, we exit with preemption
2949  *	and BH disabled, to make sure another thread or softirq handling
2950  *	wont try to get same context.
2951  */
2952 struct tcp_md5sig_pool *tcp_get_md5sig_pool(void)
2953 {
2954 	struct tcp_md5sig_pool __percpu *p;
2955 
2956 	local_bh_disable();
2957 	p = ACCESS_ONCE(tcp_md5sig_pool);
2958 	if (p)
2959 		return __this_cpu_ptr(p);
2960 
2961 	local_bh_enable();
2962 	return NULL;
2963 }
2964 EXPORT_SYMBOL(tcp_get_md5sig_pool);
2965 
2966 int tcp_md5_hash_header(struct tcp_md5sig_pool *hp,
2967 			const struct tcphdr *th)
2968 {
2969 	struct scatterlist sg;
2970 	struct tcphdr hdr;
2971 	int err;
2972 
2973 	/* We are not allowed to change tcphdr, make a local copy */
2974 	memcpy(&hdr, th, sizeof(hdr));
2975 	hdr.check = 0;
2976 
2977 	/* options aren't included in the hash */
2978 	sg_init_one(&sg, &hdr, sizeof(hdr));
2979 	err = crypto_hash_update(&hp->md5_desc, &sg, sizeof(hdr));
2980 	return err;
2981 }
2982 EXPORT_SYMBOL(tcp_md5_hash_header);
2983 
2984 int tcp_md5_hash_skb_data(struct tcp_md5sig_pool *hp,
2985 			  const struct sk_buff *skb, unsigned int header_len)
2986 {
2987 	struct scatterlist sg;
2988 	const struct tcphdr *tp = tcp_hdr(skb);
2989 	struct hash_desc *desc = &hp->md5_desc;
2990 	unsigned int i;
2991 	const unsigned int head_data_len = skb_headlen(skb) > header_len ?
2992 					   skb_headlen(skb) - header_len : 0;
2993 	const struct skb_shared_info *shi = skb_shinfo(skb);
2994 	struct sk_buff *frag_iter;
2995 
2996 	sg_init_table(&sg, 1);
2997 
2998 	sg_set_buf(&sg, ((u8 *) tp) + header_len, head_data_len);
2999 	if (crypto_hash_update(desc, &sg, head_data_len))
3000 		return 1;
3001 
3002 	for (i = 0; i < shi->nr_frags; ++i) {
3003 		const struct skb_frag_struct *f = &shi->frags[i];
3004 		unsigned int offset = f->page_offset;
3005 		struct page *page = skb_frag_page(f) + (offset >> PAGE_SHIFT);
3006 
3007 		sg_set_page(&sg, page, skb_frag_size(f),
3008 			    offset_in_page(offset));
3009 		if (crypto_hash_update(desc, &sg, skb_frag_size(f)))
3010 			return 1;
3011 	}
3012 
3013 	skb_walk_frags(skb, frag_iter)
3014 		if (tcp_md5_hash_skb_data(hp, frag_iter, 0))
3015 			return 1;
3016 
3017 	return 0;
3018 }
3019 EXPORT_SYMBOL(tcp_md5_hash_skb_data);
3020 
3021 int tcp_md5_hash_key(struct tcp_md5sig_pool *hp, const struct tcp_md5sig_key *key)
3022 {
3023 	struct scatterlist sg;
3024 
3025 	sg_init_one(&sg, key->key, key->keylen);
3026 	return crypto_hash_update(&hp->md5_desc, &sg, key->keylen);
3027 }
3028 EXPORT_SYMBOL(tcp_md5_hash_key);
3029 
3030 #endif
3031 
3032 void tcp_done(struct sock *sk)
3033 {
3034 	struct request_sock *req = tcp_sk(sk)->fastopen_rsk;
3035 
3036 	if (sk->sk_state == TCP_SYN_SENT || sk->sk_state == TCP_SYN_RECV)
3037 		TCP_INC_STATS_BH(sock_net(sk), TCP_MIB_ATTEMPTFAILS);
3038 
3039 	tcp_set_state(sk, TCP_CLOSE);
3040 	tcp_clear_xmit_timers(sk);
3041 	if (req != NULL)
3042 		reqsk_fastopen_remove(sk, req, false);
3043 
3044 	sk->sk_shutdown = SHUTDOWN_MASK;
3045 
3046 	if (!sock_flag(sk, SOCK_DEAD))
3047 		sk->sk_state_change(sk);
3048 	else
3049 		inet_csk_destroy_sock(sk);
3050 }
3051 EXPORT_SYMBOL_GPL(tcp_done);
3052 
3053 extern struct tcp_congestion_ops tcp_reno;
3054 
3055 static __initdata unsigned long thash_entries;
3056 static int __init set_thash_entries(char *str)
3057 {
3058 	ssize_t ret;
3059 
3060 	if (!str)
3061 		return 0;
3062 
3063 	ret = kstrtoul(str, 0, &thash_entries);
3064 	if (ret)
3065 		return 0;
3066 
3067 	return 1;
3068 }
3069 __setup("thash_entries=", set_thash_entries);
3070 
3071 void tcp_init_mem(struct net *net)
3072 {
3073 	unsigned long limit = nr_free_buffer_pages() / 8;
3074 	limit = max(limit, 128UL);
3075 	net->ipv4.sysctl_tcp_mem[0] = limit / 4 * 3;
3076 	net->ipv4.sysctl_tcp_mem[1] = limit;
3077 	net->ipv4.sysctl_tcp_mem[2] = net->ipv4.sysctl_tcp_mem[0] * 2;
3078 }
3079 
3080 void __init tcp_init(void)
3081 {
3082 	struct sk_buff *skb = NULL;
3083 	unsigned long limit;
3084 	int max_rshare, max_wshare, cnt;
3085 	unsigned int i;
3086 
3087 	BUILD_BUG_ON(sizeof(struct tcp_skb_cb) > sizeof(skb->cb));
3088 
3089 	percpu_counter_init(&tcp_sockets_allocated, 0);
3090 	percpu_counter_init(&tcp_orphan_count, 0);
3091 	tcp_hashinfo.bind_bucket_cachep =
3092 		kmem_cache_create("tcp_bind_bucket",
3093 				  sizeof(struct inet_bind_bucket), 0,
3094 				  SLAB_HWCACHE_ALIGN|SLAB_PANIC, NULL);
3095 
3096 	/* Size and allocate the main established and bind bucket
3097 	 * hash tables.
3098 	 *
3099 	 * The methodology is similar to that of the buffer cache.
3100 	 */
3101 	tcp_hashinfo.ehash =
3102 		alloc_large_system_hash("TCP established",
3103 					sizeof(struct inet_ehash_bucket),
3104 					thash_entries,
3105 					17, /* one slot per 128 KB of memory */
3106 					0,
3107 					NULL,
3108 					&tcp_hashinfo.ehash_mask,
3109 					0,
3110 					thash_entries ? 0 : 512 * 1024);
3111 	for (i = 0; i <= tcp_hashinfo.ehash_mask; i++) {
3112 		INIT_HLIST_NULLS_HEAD(&tcp_hashinfo.ehash[i].chain, i);
3113 		INIT_HLIST_NULLS_HEAD(&tcp_hashinfo.ehash[i].twchain, i);
3114 	}
3115 	if (inet_ehash_locks_alloc(&tcp_hashinfo))
3116 		panic("TCP: failed to alloc ehash_locks");
3117 	tcp_hashinfo.bhash =
3118 		alloc_large_system_hash("TCP bind",
3119 					sizeof(struct inet_bind_hashbucket),
3120 					tcp_hashinfo.ehash_mask + 1,
3121 					17, /* one slot per 128 KB of memory */
3122 					0,
3123 					&tcp_hashinfo.bhash_size,
3124 					NULL,
3125 					0,
3126 					64 * 1024);
3127 	tcp_hashinfo.bhash_size = 1U << tcp_hashinfo.bhash_size;
3128 	for (i = 0; i < tcp_hashinfo.bhash_size; i++) {
3129 		spin_lock_init(&tcp_hashinfo.bhash[i].lock);
3130 		INIT_HLIST_HEAD(&tcp_hashinfo.bhash[i].chain);
3131 	}
3132 
3133 
3134 	cnt = tcp_hashinfo.ehash_mask + 1;
3135 
3136 	tcp_death_row.sysctl_max_tw_buckets = cnt / 2;
3137 	sysctl_tcp_max_orphans = cnt / 2;
3138 	sysctl_max_syn_backlog = max(128, cnt / 256);
3139 
3140 	tcp_init_mem(&init_net);
3141 	/* Set per-socket limits to no more than 1/128 the pressure threshold */
3142 	limit = nr_free_buffer_pages() << (PAGE_SHIFT - 7);
3143 	max_wshare = min(4UL*1024*1024, limit);
3144 	max_rshare = min(6UL*1024*1024, limit);
3145 
3146 	sysctl_tcp_wmem[0] = SK_MEM_QUANTUM;
3147 	sysctl_tcp_wmem[1] = 16*1024;
3148 	sysctl_tcp_wmem[2] = max(64*1024, max_wshare);
3149 
3150 	sysctl_tcp_rmem[0] = SK_MEM_QUANTUM;
3151 	sysctl_tcp_rmem[1] = 87380;
3152 	sysctl_tcp_rmem[2] = max(87380, max_rshare);
3153 
3154 	pr_info("Hash tables configured (established %u bind %u)\n",
3155 		tcp_hashinfo.ehash_mask + 1, tcp_hashinfo.bhash_size);
3156 
3157 	tcp_metrics_init();
3158 
3159 	tcp_register_congestion_control(&tcp_reno);
3160 
3161 	tcp_tasklet_init();
3162 }
3163