xref: /openbmc/linux/net/ipv4/tcp.c (revision e868d61272caa648214046a096e5a6bfc068dc8c)
1 /*
2  * INET		An implementation of the TCP/IP protocol suite for the LINUX
3  *		operating system.  INET is implemented using the  BSD Socket
4  *		interface as the means of communication with the user level.
5  *
6  *		Implementation of the Transmission Control Protocol(TCP).
7  *
8  * Version:	$Id: tcp.c,v 1.216 2002/02/01 22:01:04 davem Exp $
9  *
10  * Authors:	Ross Biro
11  *		Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
12  *		Mark Evans, <evansmp@uhura.aston.ac.uk>
13  *		Corey Minyard <wf-rch!minyard@relay.EU.net>
14  *		Florian La Roche, <flla@stud.uni-sb.de>
15  *		Charles Hedrick, <hedrick@klinzhai.rutgers.edu>
16  *		Linus Torvalds, <torvalds@cs.helsinki.fi>
17  *		Alan Cox, <gw4pts@gw4pts.ampr.org>
18  *		Matthew Dillon, <dillon@apollo.west.oic.com>
19  *		Arnt Gulbrandsen, <agulbra@nvg.unit.no>
20  *		Jorge Cwik, <jorge@laser.satlink.net>
21  *
22  * Fixes:
23  *		Alan Cox	:	Numerous verify_area() calls
24  *		Alan Cox	:	Set the ACK bit on a reset
25  *		Alan Cox	:	Stopped it crashing if it closed while
26  *					sk->inuse=1 and was trying to connect
27  *					(tcp_err()).
28  *		Alan Cox	:	All icmp error handling was broken
29  *					pointers passed where wrong and the
30  *					socket was looked up backwards. Nobody
31  *					tested any icmp error code obviously.
32  *		Alan Cox	:	tcp_err() now handled properly. It
33  *					wakes people on errors. poll
34  *					behaves and the icmp error race
35  *					has gone by moving it into sock.c
36  *		Alan Cox	:	tcp_send_reset() fixed to work for
37  *					everything not just packets for
38  *					unknown sockets.
39  *		Alan Cox	:	tcp option processing.
40  *		Alan Cox	:	Reset tweaked (still not 100%) [Had
41  *					syn rule wrong]
42  *		Herp Rosmanith  :	More reset fixes
43  *		Alan Cox	:	No longer acks invalid rst frames.
44  *					Acking any kind of RST is right out.
45  *		Alan Cox	:	Sets an ignore me flag on an rst
46  *					receive otherwise odd bits of prattle
47  *					escape still
48  *		Alan Cox	:	Fixed another acking RST frame bug.
49  *					Should stop LAN workplace lockups.
50  *		Alan Cox	: 	Some tidyups using the new skb list
51  *					facilities
52  *		Alan Cox	:	sk->keepopen now seems to work
53  *		Alan Cox	:	Pulls options out correctly on accepts
54  *		Alan Cox	:	Fixed assorted sk->rqueue->next errors
55  *		Alan Cox	:	PSH doesn't end a TCP read. Switched a
56  *					bit to skb ops.
57  *		Alan Cox	:	Tidied tcp_data to avoid a potential
58  *					nasty.
59  *		Alan Cox	:	Added some better commenting, as the
60  *					tcp is hard to follow
61  *		Alan Cox	:	Removed incorrect check for 20 * psh
62  *	Michael O'Reilly	:	ack < copied bug fix.
63  *	Johannes Stille		:	Misc tcp fixes (not all in yet).
64  *		Alan Cox	:	FIN with no memory -> CRASH
65  *		Alan Cox	:	Added socket option proto entries.
66  *					Also added awareness of them to accept.
67  *		Alan Cox	:	Added TCP options (SOL_TCP)
68  *		Alan Cox	:	Switched wakeup calls to callbacks,
69  *					so the kernel can layer network
70  *					sockets.
71  *		Alan Cox	:	Use ip_tos/ip_ttl settings.
72  *		Alan Cox	:	Handle FIN (more) properly (we hope).
73  *		Alan Cox	:	RST frames sent on unsynchronised
74  *					state ack error.
75  *		Alan Cox	:	Put in missing check for SYN bit.
76  *		Alan Cox	:	Added tcp_select_window() aka NET2E
77  *					window non shrink trick.
78  *		Alan Cox	:	Added a couple of small NET2E timer
79  *					fixes
80  *		Charles Hedrick :	TCP fixes
81  *		Toomas Tamm	:	TCP window fixes
82  *		Alan Cox	:	Small URG fix to rlogin ^C ack fight
83  *		Charles Hedrick	:	Rewrote most of it to actually work
84  *		Linus		:	Rewrote tcp_read() and URG handling
85  *					completely
86  *		Gerhard Koerting:	Fixed some missing timer handling
87  *		Matthew Dillon  :	Reworked TCP machine states as per RFC
88  *		Gerhard Koerting:	PC/TCP workarounds
89  *		Adam Caldwell	:	Assorted timer/timing errors
90  *		Matthew Dillon	:	Fixed another RST bug
91  *		Alan Cox	:	Move to kernel side addressing changes.
92  *		Alan Cox	:	Beginning work on TCP fastpathing
93  *					(not yet usable)
94  *		Arnt Gulbrandsen:	Turbocharged tcp_check() routine.
95  *		Alan Cox	:	TCP fast path debugging
96  *		Alan Cox	:	Window clamping
97  *		Michael Riepe	:	Bug in tcp_check()
98  *		Matt Dillon	:	More TCP improvements and RST bug fixes
99  *		Matt Dillon	:	Yet more small nasties remove from the
100  *					TCP code (Be very nice to this man if
101  *					tcp finally works 100%) 8)
102  *		Alan Cox	:	BSD accept semantics.
103  *		Alan Cox	:	Reset on closedown bug.
104  *	Peter De Schrijver	:	ENOTCONN check missing in tcp_sendto().
105  *		Michael Pall	:	Handle poll() after URG properly in
106  *					all cases.
107  *		Michael Pall	:	Undo the last fix in tcp_read_urg()
108  *					(multi URG PUSH broke rlogin).
109  *		Michael Pall	:	Fix the multi URG PUSH problem in
110  *					tcp_readable(), poll() after URG
111  *					works now.
112  *		Michael Pall	:	recv(...,MSG_OOB) never blocks in the
113  *					BSD api.
114  *		Alan Cox	:	Changed the semantics of sk->socket to
115  *					fix a race and a signal problem with
116  *					accept() and async I/O.
117  *		Alan Cox	:	Relaxed the rules on tcp_sendto().
118  *		Yury Shevchuk	:	Really fixed accept() blocking problem.
119  *		Craig I. Hagan  :	Allow for BSD compatible TIME_WAIT for
120  *					clients/servers which listen in on
121  *					fixed ports.
122  *		Alan Cox	:	Cleaned the above up and shrank it to
123  *					a sensible code size.
124  *		Alan Cox	:	Self connect lockup fix.
125  *		Alan Cox	:	No connect to multicast.
126  *		Ross Biro	:	Close unaccepted children on master
127  *					socket close.
128  *		Alan Cox	:	Reset tracing code.
129  *		Alan Cox	:	Spurious resets on shutdown.
130  *		Alan Cox	:	Giant 15 minute/60 second timer error
131  *		Alan Cox	:	Small whoops in polling before an
132  *					accept.
133  *		Alan Cox	:	Kept the state trace facility since
134  *					it's handy for debugging.
135  *		Alan Cox	:	More reset handler fixes.
136  *		Alan Cox	:	Started rewriting the code based on
137  *					the RFC's for other useful protocol
138  *					references see: Comer, KA9Q NOS, and
139  *					for a reference on the difference
140  *					between specifications and how BSD
141  *					works see the 4.4lite source.
142  *		A.N.Kuznetsov	:	Don't time wait on completion of tidy
143  *					close.
144  *		Linus Torvalds	:	Fin/Shutdown & copied_seq changes.
145  *		Linus Torvalds	:	Fixed BSD port reuse to work first syn
146  *		Alan Cox	:	Reimplemented timers as per the RFC
147  *					and using multiple timers for sanity.
148  *		Alan Cox	:	Small bug fixes, and a lot of new
149  *					comments.
150  *		Alan Cox	:	Fixed dual reader crash by locking
151  *					the buffers (much like datagram.c)
152  *		Alan Cox	:	Fixed stuck sockets in probe. A probe
153  *					now gets fed up of retrying without
154  *					(even a no space) answer.
155  *		Alan Cox	:	Extracted closing code better
156  *		Alan Cox	:	Fixed the closing state machine to
157  *					resemble the RFC.
158  *		Alan Cox	:	More 'per spec' fixes.
159  *		Jorge Cwik	:	Even faster checksumming.
160  *		Alan Cox	:	tcp_data() doesn't ack illegal PSH
161  *					only frames. At least one pc tcp stack
162  *					generates them.
163  *		Alan Cox	:	Cache last socket.
164  *		Alan Cox	:	Per route irtt.
165  *		Matt Day	:	poll()->select() match BSD precisely on error
166  *		Alan Cox	:	New buffers
167  *		Marc Tamsky	:	Various sk->prot->retransmits and
168  *					sk->retransmits misupdating fixed.
169  *					Fixed tcp_write_timeout: stuck close,
170  *					and TCP syn retries gets used now.
171  *		Mark Yarvis	:	In tcp_read_wakeup(), don't send an
172  *					ack if state is TCP_CLOSED.
173  *		Alan Cox	:	Look up device on a retransmit - routes may
174  *					change. Doesn't yet cope with MSS shrink right
175  *					but it's a start!
176  *		Marc Tamsky	:	Closing in closing fixes.
177  *		Mike Shaver	:	RFC1122 verifications.
178  *		Alan Cox	:	rcv_saddr errors.
179  *		Alan Cox	:	Block double connect().
180  *		Alan Cox	:	Small hooks for enSKIP.
181  *		Alexey Kuznetsov:	Path MTU discovery.
182  *		Alan Cox	:	Support soft errors.
183  *		Alan Cox	:	Fix MTU discovery pathological case
184  *					when the remote claims no mtu!
185  *		Marc Tamsky	:	TCP_CLOSE fix.
186  *		Colin (G3TNE)	:	Send a reset on syn ack replies in
187  *					window but wrong (fixes NT lpd problems)
188  *		Pedro Roque	:	Better TCP window handling, delayed ack.
189  *		Joerg Reuter	:	No modification of locked buffers in
190  *					tcp_do_retransmit()
191  *		Eric Schenk	:	Changed receiver side silly window
192  *					avoidance algorithm to BSD style
193  *					algorithm. This doubles throughput
194  *					against machines running Solaris,
195  *					and seems to result in general
196  *					improvement.
197  *	Stefan Magdalinski	:	adjusted tcp_readable() to fix FIONREAD
198  *	Willy Konynenberg	:	Transparent proxying support.
199  *	Mike McLagan		:	Routing by source
200  *		Keith Owens	:	Do proper merging with partial SKB's in
201  *					tcp_do_sendmsg to avoid burstiness.
202  *		Eric Schenk	:	Fix fast close down bug with
203  *					shutdown() followed by close().
204  *		Andi Kleen 	:	Make poll agree with SIGIO
205  *	Salvatore Sanfilippo	:	Support SO_LINGER with linger == 1 and
206  *					lingertime == 0 (RFC 793 ABORT Call)
207  *	Hirokazu Takahashi	:	Use copy_from_user() instead of
208  *					csum_and_copy_from_user() if possible.
209  *
210  *		This program is free software; you can redistribute it and/or
211  *		modify it under the terms of the GNU General Public License
212  *		as published by the Free Software Foundation; either version
213  *		2 of the License, or(at your option) any later version.
214  *
215  * Description of States:
216  *
217  *	TCP_SYN_SENT		sent a connection request, waiting for ack
218  *
219  *	TCP_SYN_RECV		received a connection request, sent ack,
220  *				waiting for final ack in three-way handshake.
221  *
222  *	TCP_ESTABLISHED		connection established
223  *
224  *	TCP_FIN_WAIT1		our side has shutdown, waiting to complete
225  *				transmission of remaining buffered data
226  *
227  *	TCP_FIN_WAIT2		all buffered data sent, waiting for remote
228  *				to shutdown
229  *
230  *	TCP_CLOSING		both sides have shutdown but we still have
231  *				data we have to finish sending
232  *
233  *	TCP_TIME_WAIT		timeout to catch resent junk before entering
234  *				closed, can only be entered from FIN_WAIT2
235  *				or CLOSING.  Required because the other end
236  *				may not have gotten our last ACK causing it
237  *				to retransmit the data packet (which we ignore)
238  *
239  *	TCP_CLOSE_WAIT		remote side has shutdown and is waiting for
240  *				us to finish writing our data and to shutdown
241  *				(we have to close() to move on to LAST_ACK)
242  *
243  *	TCP_LAST_ACK		out side has shutdown after remote has
244  *				shutdown.  There may still be data in our
245  *				buffer that we have to finish sending
246  *
247  *	TCP_CLOSE		socket is finished
248  */
249 
250 #include <linux/module.h>
251 #include <linux/types.h>
252 #include <linux/fcntl.h>
253 #include <linux/poll.h>
254 #include <linux/init.h>
255 #include <linux/fs.h>
256 #include <linux/random.h>
257 #include <linux/bootmem.h>
258 #include <linux/cache.h>
259 #include <linux/err.h>
260 #include <linux/crypto.h>
261 
262 #include <net/icmp.h>
263 #include <net/tcp.h>
264 #include <net/xfrm.h>
265 #include <net/ip.h>
266 #include <net/netdma.h>
267 
268 #include <asm/uaccess.h>
269 #include <asm/ioctls.h>
270 
271 int sysctl_tcp_fin_timeout __read_mostly = TCP_FIN_TIMEOUT;
272 
273 DEFINE_SNMP_STAT(struct tcp_mib, tcp_statistics) __read_mostly;
274 
275 atomic_t tcp_orphan_count = ATOMIC_INIT(0);
276 
277 EXPORT_SYMBOL_GPL(tcp_orphan_count);
278 
279 int sysctl_tcp_mem[3] __read_mostly;
280 int sysctl_tcp_wmem[3] __read_mostly;
281 int sysctl_tcp_rmem[3] __read_mostly;
282 
283 EXPORT_SYMBOL(sysctl_tcp_mem);
284 EXPORT_SYMBOL(sysctl_tcp_rmem);
285 EXPORT_SYMBOL(sysctl_tcp_wmem);
286 
287 atomic_t tcp_memory_allocated;	/* Current allocated memory. */
288 atomic_t tcp_sockets_allocated;	/* Current number of TCP sockets. */
289 
290 EXPORT_SYMBOL(tcp_memory_allocated);
291 EXPORT_SYMBOL(tcp_sockets_allocated);
292 
293 /*
294  * Pressure flag: try to collapse.
295  * Technical note: it is used by multiple contexts non atomically.
296  * All the sk_stream_mem_schedule() is of this nature: accounting
297  * is strict, actions are advisory and have some latency.
298  */
299 int tcp_memory_pressure __read_mostly;
300 
301 EXPORT_SYMBOL(tcp_memory_pressure);
302 
303 void tcp_enter_memory_pressure(void)
304 {
305 	if (!tcp_memory_pressure) {
306 		NET_INC_STATS(LINUX_MIB_TCPMEMORYPRESSURES);
307 		tcp_memory_pressure = 1;
308 	}
309 }
310 
311 EXPORT_SYMBOL(tcp_enter_memory_pressure);
312 
313 /*
314  *	Wait for a TCP event.
315  *
316  *	Note that we don't need to lock the socket, as the upper poll layers
317  *	take care of normal races (between the test and the event) and we don't
318  *	go look at any of the socket buffers directly.
319  */
320 unsigned int tcp_poll(struct file *file, struct socket *sock, poll_table *wait)
321 {
322 	unsigned int mask;
323 	struct sock *sk = sock->sk;
324 	struct tcp_sock *tp = tcp_sk(sk);
325 
326 	poll_wait(file, sk->sk_sleep, wait);
327 	if (sk->sk_state == TCP_LISTEN)
328 		return inet_csk_listen_poll(sk);
329 
330 	/* Socket is not locked. We are protected from async events
331 	   by poll logic and correct handling of state changes
332 	   made by another threads is impossible in any case.
333 	 */
334 
335 	mask = 0;
336 	if (sk->sk_err)
337 		mask = POLLERR;
338 
339 	/*
340 	 * POLLHUP is certainly not done right. But poll() doesn't
341 	 * have a notion of HUP in just one direction, and for a
342 	 * socket the read side is more interesting.
343 	 *
344 	 * Some poll() documentation says that POLLHUP is incompatible
345 	 * with the POLLOUT/POLLWR flags, so somebody should check this
346 	 * all. But careful, it tends to be safer to return too many
347 	 * bits than too few, and you can easily break real applications
348 	 * if you don't tell them that something has hung up!
349 	 *
350 	 * Check-me.
351 	 *
352 	 * Check number 1. POLLHUP is _UNMASKABLE_ event (see UNIX98 and
353 	 * our fs/select.c). It means that after we received EOF,
354 	 * poll always returns immediately, making impossible poll() on write()
355 	 * in state CLOSE_WAIT. One solution is evident --- to set POLLHUP
356 	 * if and only if shutdown has been made in both directions.
357 	 * Actually, it is interesting to look how Solaris and DUX
358 	 * solve this dilemma. I would prefer, if PULLHUP were maskable,
359 	 * then we could set it on SND_SHUTDOWN. BTW examples given
360 	 * in Stevens' books assume exactly this behaviour, it explains
361 	 * why PULLHUP is incompatible with POLLOUT.	--ANK
362 	 *
363 	 * NOTE. Check for TCP_CLOSE is added. The goal is to prevent
364 	 * blocking on fresh not-connected or disconnected socket. --ANK
365 	 */
366 	if (sk->sk_shutdown == SHUTDOWN_MASK || sk->sk_state == TCP_CLOSE)
367 		mask |= POLLHUP;
368 	if (sk->sk_shutdown & RCV_SHUTDOWN)
369 		mask |= POLLIN | POLLRDNORM | POLLRDHUP;
370 
371 	/* Connected? */
372 	if ((1 << sk->sk_state) & ~(TCPF_SYN_SENT | TCPF_SYN_RECV)) {
373 		/* Potential race condition. If read of tp below will
374 		 * escape above sk->sk_state, we can be illegally awaken
375 		 * in SYN_* states. */
376 		if ((tp->rcv_nxt != tp->copied_seq) &&
377 		    (tp->urg_seq != tp->copied_seq ||
378 		     tp->rcv_nxt != tp->copied_seq + 1 ||
379 		     sock_flag(sk, SOCK_URGINLINE) || !tp->urg_data))
380 			mask |= POLLIN | POLLRDNORM;
381 
382 		if (!(sk->sk_shutdown & SEND_SHUTDOWN)) {
383 			if (sk_stream_wspace(sk) >= sk_stream_min_wspace(sk)) {
384 				mask |= POLLOUT | POLLWRNORM;
385 			} else {  /* send SIGIO later */
386 				set_bit(SOCK_ASYNC_NOSPACE,
387 					&sk->sk_socket->flags);
388 				set_bit(SOCK_NOSPACE, &sk->sk_socket->flags);
389 
390 				/* Race breaker. If space is freed after
391 				 * wspace test but before the flags are set,
392 				 * IO signal will be lost.
393 				 */
394 				if (sk_stream_wspace(sk) >= sk_stream_min_wspace(sk))
395 					mask |= POLLOUT | POLLWRNORM;
396 			}
397 		}
398 
399 		if (tp->urg_data & TCP_URG_VALID)
400 			mask |= POLLPRI;
401 	}
402 	return mask;
403 }
404 
405 int tcp_ioctl(struct sock *sk, int cmd, unsigned long arg)
406 {
407 	struct tcp_sock *tp = tcp_sk(sk);
408 	int answ;
409 
410 	switch (cmd) {
411 	case SIOCINQ:
412 		if (sk->sk_state == TCP_LISTEN)
413 			return -EINVAL;
414 
415 		lock_sock(sk);
416 		if ((1 << sk->sk_state) & (TCPF_SYN_SENT | TCPF_SYN_RECV))
417 			answ = 0;
418 		else if (sock_flag(sk, SOCK_URGINLINE) ||
419 			 !tp->urg_data ||
420 			 before(tp->urg_seq, tp->copied_seq) ||
421 			 !before(tp->urg_seq, tp->rcv_nxt)) {
422 			answ = tp->rcv_nxt - tp->copied_seq;
423 
424 			/* Subtract 1, if FIN is in queue. */
425 			if (answ && !skb_queue_empty(&sk->sk_receive_queue))
426 				answ -=
427 		       tcp_hdr((struct sk_buff *)sk->sk_receive_queue.prev)->fin;
428 		} else
429 			answ = tp->urg_seq - tp->copied_seq;
430 		release_sock(sk);
431 		break;
432 	case SIOCATMARK:
433 		answ = tp->urg_data && tp->urg_seq == tp->copied_seq;
434 		break;
435 	case SIOCOUTQ:
436 		if (sk->sk_state == TCP_LISTEN)
437 			return -EINVAL;
438 
439 		if ((1 << sk->sk_state) & (TCPF_SYN_SENT | TCPF_SYN_RECV))
440 			answ = 0;
441 		else
442 			answ = tp->write_seq - tp->snd_una;
443 		break;
444 	default:
445 		return -ENOIOCTLCMD;
446 	}
447 
448 	return put_user(answ, (int __user *)arg);
449 }
450 
451 static inline void tcp_mark_push(struct tcp_sock *tp, struct sk_buff *skb)
452 {
453 	TCP_SKB_CB(skb)->flags |= TCPCB_FLAG_PSH;
454 	tp->pushed_seq = tp->write_seq;
455 }
456 
457 static inline int forced_push(struct tcp_sock *tp)
458 {
459 	return after(tp->write_seq, tp->pushed_seq + (tp->max_window >> 1));
460 }
461 
462 static inline void skb_entail(struct sock *sk, struct sk_buff *skb)
463 {
464 	struct tcp_sock *tp = tcp_sk(sk);
465 	struct tcp_skb_cb *tcb = TCP_SKB_CB(skb);
466 
467 	skb->csum    = 0;
468 	tcb->seq     = tcb->end_seq = tp->write_seq;
469 	tcb->flags   = TCPCB_FLAG_ACK;
470 	tcb->sacked  = 0;
471 	skb_header_release(skb);
472 	tcp_add_write_queue_tail(sk, skb);
473 	sk_charge_skb(sk, skb);
474 	if (tp->nonagle & TCP_NAGLE_PUSH)
475 		tp->nonagle &= ~TCP_NAGLE_PUSH;
476 }
477 
478 static inline void tcp_mark_urg(struct tcp_sock *tp, int flags,
479 				struct sk_buff *skb)
480 {
481 	if (flags & MSG_OOB) {
482 		tp->urg_mode = 1;
483 		tp->snd_up = tp->write_seq;
484 		TCP_SKB_CB(skb)->sacked |= TCPCB_URG;
485 	}
486 }
487 
488 static inline void tcp_push(struct sock *sk, int flags, int mss_now,
489 			    int nonagle)
490 {
491 	struct tcp_sock *tp = tcp_sk(sk);
492 
493 	if (tcp_send_head(sk)) {
494 		struct sk_buff *skb = tcp_write_queue_tail(sk);
495 		if (!(flags & MSG_MORE) || forced_push(tp))
496 			tcp_mark_push(tp, skb);
497 		tcp_mark_urg(tp, flags, skb);
498 		__tcp_push_pending_frames(sk, mss_now,
499 					  (flags & MSG_MORE) ? TCP_NAGLE_CORK : nonagle);
500 	}
501 }
502 
503 static ssize_t do_tcp_sendpages(struct sock *sk, struct page **pages, int poffset,
504 			 size_t psize, int flags)
505 {
506 	struct tcp_sock *tp = tcp_sk(sk);
507 	int mss_now, size_goal;
508 	int err;
509 	ssize_t copied;
510 	long timeo = sock_sndtimeo(sk, flags & MSG_DONTWAIT);
511 
512 	/* Wait for a connection to finish. */
513 	if ((1 << sk->sk_state) & ~(TCPF_ESTABLISHED | TCPF_CLOSE_WAIT))
514 		if ((err = sk_stream_wait_connect(sk, &timeo)) != 0)
515 			goto out_err;
516 
517 	clear_bit(SOCK_ASYNC_NOSPACE, &sk->sk_socket->flags);
518 
519 	mss_now = tcp_current_mss(sk, !(flags&MSG_OOB));
520 	size_goal = tp->xmit_size_goal;
521 	copied = 0;
522 
523 	err = -EPIPE;
524 	if (sk->sk_err || (sk->sk_shutdown & SEND_SHUTDOWN))
525 		goto do_error;
526 
527 	while (psize > 0) {
528 		struct sk_buff *skb = tcp_write_queue_tail(sk);
529 		struct page *page = pages[poffset / PAGE_SIZE];
530 		int copy, i, can_coalesce;
531 		int offset = poffset % PAGE_SIZE;
532 		int size = min_t(size_t, psize, PAGE_SIZE - offset);
533 
534 		if (!tcp_send_head(sk) || (copy = size_goal - skb->len) <= 0) {
535 new_segment:
536 			if (!sk_stream_memory_free(sk))
537 				goto wait_for_sndbuf;
538 
539 			skb = sk_stream_alloc_pskb(sk, 0, 0,
540 						   sk->sk_allocation);
541 			if (!skb)
542 				goto wait_for_memory;
543 
544 			skb_entail(sk, skb);
545 			copy = size_goal;
546 		}
547 
548 		if (copy > size)
549 			copy = size;
550 
551 		i = skb_shinfo(skb)->nr_frags;
552 		can_coalesce = skb_can_coalesce(skb, i, page, offset);
553 		if (!can_coalesce && i >= MAX_SKB_FRAGS) {
554 			tcp_mark_push(tp, skb);
555 			goto new_segment;
556 		}
557 		if (!sk_stream_wmem_schedule(sk, copy))
558 			goto wait_for_memory;
559 
560 		if (can_coalesce) {
561 			skb_shinfo(skb)->frags[i - 1].size += copy;
562 		} else {
563 			get_page(page);
564 			skb_fill_page_desc(skb, i, page, offset, copy);
565 		}
566 
567 		skb->len += copy;
568 		skb->data_len += copy;
569 		skb->truesize += copy;
570 		sk->sk_wmem_queued += copy;
571 		sk->sk_forward_alloc -= copy;
572 		skb->ip_summed = CHECKSUM_PARTIAL;
573 		tp->write_seq += copy;
574 		TCP_SKB_CB(skb)->end_seq += copy;
575 		skb_shinfo(skb)->gso_segs = 0;
576 
577 		if (!copied)
578 			TCP_SKB_CB(skb)->flags &= ~TCPCB_FLAG_PSH;
579 
580 		copied += copy;
581 		poffset += copy;
582 		if (!(psize -= copy))
583 			goto out;
584 
585 		if (skb->len < mss_now || (flags & MSG_OOB))
586 			continue;
587 
588 		if (forced_push(tp)) {
589 			tcp_mark_push(tp, skb);
590 			__tcp_push_pending_frames(sk, mss_now, TCP_NAGLE_PUSH);
591 		} else if (skb == tcp_send_head(sk))
592 			tcp_push_one(sk, mss_now);
593 		continue;
594 
595 wait_for_sndbuf:
596 		set_bit(SOCK_NOSPACE, &sk->sk_socket->flags);
597 wait_for_memory:
598 		if (copied)
599 			tcp_push(sk, flags & ~MSG_MORE, mss_now, TCP_NAGLE_PUSH);
600 
601 		if ((err = sk_stream_wait_memory(sk, &timeo)) != 0)
602 			goto do_error;
603 
604 		mss_now = tcp_current_mss(sk, !(flags&MSG_OOB));
605 		size_goal = tp->xmit_size_goal;
606 	}
607 
608 out:
609 	if (copied)
610 		tcp_push(sk, flags, mss_now, tp->nonagle);
611 	return copied;
612 
613 do_error:
614 	if (copied)
615 		goto out;
616 out_err:
617 	return sk_stream_error(sk, flags, err);
618 }
619 
620 ssize_t tcp_sendpage(struct socket *sock, struct page *page, int offset,
621 		     size_t size, int flags)
622 {
623 	ssize_t res;
624 	struct sock *sk = sock->sk;
625 
626 	if (!(sk->sk_route_caps & NETIF_F_SG) ||
627 	    !(sk->sk_route_caps & NETIF_F_ALL_CSUM))
628 		return sock_no_sendpage(sock, page, offset, size, flags);
629 
630 	lock_sock(sk);
631 	TCP_CHECK_TIMER(sk);
632 	res = do_tcp_sendpages(sk, &page, offset, size, flags);
633 	TCP_CHECK_TIMER(sk);
634 	release_sock(sk);
635 	return res;
636 }
637 
638 #define TCP_PAGE(sk)	(sk->sk_sndmsg_page)
639 #define TCP_OFF(sk)	(sk->sk_sndmsg_off)
640 
641 static inline int select_size(struct sock *sk)
642 {
643 	struct tcp_sock *tp = tcp_sk(sk);
644 	int tmp = tp->mss_cache;
645 
646 	if (sk->sk_route_caps & NETIF_F_SG) {
647 		if (sk_can_gso(sk))
648 			tmp = 0;
649 		else {
650 			int pgbreak = SKB_MAX_HEAD(MAX_TCP_HEADER);
651 
652 			if (tmp >= pgbreak &&
653 			    tmp <= pgbreak + (MAX_SKB_FRAGS - 1) * PAGE_SIZE)
654 				tmp = pgbreak;
655 		}
656 	}
657 
658 	return tmp;
659 }
660 
661 int tcp_sendmsg(struct kiocb *iocb, struct sock *sk, struct msghdr *msg,
662 		size_t size)
663 {
664 	struct iovec *iov;
665 	struct tcp_sock *tp = tcp_sk(sk);
666 	struct sk_buff *skb;
667 	int iovlen, flags;
668 	int mss_now, size_goal;
669 	int err, copied;
670 	long timeo;
671 
672 	lock_sock(sk);
673 	TCP_CHECK_TIMER(sk);
674 
675 	flags = msg->msg_flags;
676 	timeo = sock_sndtimeo(sk, flags & MSG_DONTWAIT);
677 
678 	/* Wait for a connection to finish. */
679 	if ((1 << sk->sk_state) & ~(TCPF_ESTABLISHED | TCPF_CLOSE_WAIT))
680 		if ((err = sk_stream_wait_connect(sk, &timeo)) != 0)
681 			goto out_err;
682 
683 	/* This should be in poll */
684 	clear_bit(SOCK_ASYNC_NOSPACE, &sk->sk_socket->flags);
685 
686 	mss_now = tcp_current_mss(sk, !(flags&MSG_OOB));
687 	size_goal = tp->xmit_size_goal;
688 
689 	/* Ok commence sending. */
690 	iovlen = msg->msg_iovlen;
691 	iov = msg->msg_iov;
692 	copied = 0;
693 
694 	err = -EPIPE;
695 	if (sk->sk_err || (sk->sk_shutdown & SEND_SHUTDOWN))
696 		goto do_error;
697 
698 	while (--iovlen >= 0) {
699 		int seglen = iov->iov_len;
700 		unsigned char __user *from = iov->iov_base;
701 
702 		iov++;
703 
704 		while (seglen > 0) {
705 			int copy;
706 
707 			skb = tcp_write_queue_tail(sk);
708 
709 			if (!tcp_send_head(sk) ||
710 			    (copy = size_goal - skb->len) <= 0) {
711 
712 new_segment:
713 				/* Allocate new segment. If the interface is SG,
714 				 * allocate skb fitting to single page.
715 				 */
716 				if (!sk_stream_memory_free(sk))
717 					goto wait_for_sndbuf;
718 
719 				skb = sk_stream_alloc_pskb(sk, select_size(sk),
720 							   0, sk->sk_allocation);
721 				if (!skb)
722 					goto wait_for_memory;
723 
724 				/*
725 				 * Check whether we can use HW checksum.
726 				 */
727 				if (sk->sk_route_caps & NETIF_F_ALL_CSUM)
728 					skb->ip_summed = CHECKSUM_PARTIAL;
729 
730 				skb_entail(sk, skb);
731 				copy = size_goal;
732 			}
733 
734 			/* Try to append data to the end of skb. */
735 			if (copy > seglen)
736 				copy = seglen;
737 
738 			/* Where to copy to? */
739 			if (skb_tailroom(skb) > 0) {
740 				/* We have some space in skb head. Superb! */
741 				if (copy > skb_tailroom(skb))
742 					copy = skb_tailroom(skb);
743 				if ((err = skb_add_data(skb, from, copy)) != 0)
744 					goto do_fault;
745 			} else {
746 				int merge = 0;
747 				int i = skb_shinfo(skb)->nr_frags;
748 				struct page *page = TCP_PAGE(sk);
749 				int off = TCP_OFF(sk);
750 
751 				if (skb_can_coalesce(skb, i, page, off) &&
752 				    off != PAGE_SIZE) {
753 					/* We can extend the last page
754 					 * fragment. */
755 					merge = 1;
756 				} else if (i == MAX_SKB_FRAGS ||
757 					   (!i &&
758 					   !(sk->sk_route_caps & NETIF_F_SG))) {
759 					/* Need to add new fragment and cannot
760 					 * do this because interface is non-SG,
761 					 * or because all the page slots are
762 					 * busy. */
763 					tcp_mark_push(tp, skb);
764 					goto new_segment;
765 				} else if (page) {
766 					if (off == PAGE_SIZE) {
767 						put_page(page);
768 						TCP_PAGE(sk) = page = NULL;
769 						off = 0;
770 					}
771 				} else
772 					off = 0;
773 
774 				if (copy > PAGE_SIZE - off)
775 					copy = PAGE_SIZE - off;
776 
777 				if (!sk_stream_wmem_schedule(sk, copy))
778 					goto wait_for_memory;
779 
780 				if (!page) {
781 					/* Allocate new cache page. */
782 					if (!(page = sk_stream_alloc_page(sk)))
783 						goto wait_for_memory;
784 				}
785 
786 				/* Time to copy data. We are close to
787 				 * the end! */
788 				err = skb_copy_to_page(sk, from, skb, page,
789 						       off, copy);
790 				if (err) {
791 					/* If this page was new, give it to the
792 					 * socket so it does not get leaked.
793 					 */
794 					if (!TCP_PAGE(sk)) {
795 						TCP_PAGE(sk) = page;
796 						TCP_OFF(sk) = 0;
797 					}
798 					goto do_error;
799 				}
800 
801 				/* Update the skb. */
802 				if (merge) {
803 					skb_shinfo(skb)->frags[i - 1].size +=
804 									copy;
805 				} else {
806 					skb_fill_page_desc(skb, i, page, off, copy);
807 					if (TCP_PAGE(sk)) {
808 						get_page(page);
809 					} else if (off + copy < PAGE_SIZE) {
810 						get_page(page);
811 						TCP_PAGE(sk) = page;
812 					}
813 				}
814 
815 				TCP_OFF(sk) = off + copy;
816 			}
817 
818 			if (!copied)
819 				TCP_SKB_CB(skb)->flags &= ~TCPCB_FLAG_PSH;
820 
821 			tp->write_seq += copy;
822 			TCP_SKB_CB(skb)->end_seq += copy;
823 			skb_shinfo(skb)->gso_segs = 0;
824 
825 			from += copy;
826 			copied += copy;
827 			if ((seglen -= copy) == 0 && iovlen == 0)
828 				goto out;
829 
830 			if (skb->len < mss_now || (flags & MSG_OOB))
831 				continue;
832 
833 			if (forced_push(tp)) {
834 				tcp_mark_push(tp, skb);
835 				__tcp_push_pending_frames(sk, mss_now, TCP_NAGLE_PUSH);
836 			} else if (skb == tcp_send_head(sk))
837 				tcp_push_one(sk, mss_now);
838 			continue;
839 
840 wait_for_sndbuf:
841 			set_bit(SOCK_NOSPACE, &sk->sk_socket->flags);
842 wait_for_memory:
843 			if (copied)
844 				tcp_push(sk, flags & ~MSG_MORE, mss_now, TCP_NAGLE_PUSH);
845 
846 			if ((err = sk_stream_wait_memory(sk, &timeo)) != 0)
847 				goto do_error;
848 
849 			mss_now = tcp_current_mss(sk, !(flags&MSG_OOB));
850 			size_goal = tp->xmit_size_goal;
851 		}
852 	}
853 
854 out:
855 	if (copied)
856 		tcp_push(sk, flags, mss_now, tp->nonagle);
857 	TCP_CHECK_TIMER(sk);
858 	release_sock(sk);
859 	return copied;
860 
861 do_fault:
862 	if (!skb->len) {
863 		tcp_unlink_write_queue(skb, sk);
864 		/* It is the one place in all of TCP, except connection
865 		 * reset, where we can be unlinking the send_head.
866 		 */
867 		tcp_check_send_head(sk, skb);
868 		sk_stream_free_skb(sk, skb);
869 	}
870 
871 do_error:
872 	if (copied)
873 		goto out;
874 out_err:
875 	err = sk_stream_error(sk, flags, err);
876 	TCP_CHECK_TIMER(sk);
877 	release_sock(sk);
878 	return err;
879 }
880 
881 /*
882  *	Handle reading urgent data. BSD has very simple semantics for
883  *	this, no blocking and very strange errors 8)
884  */
885 
886 static int tcp_recv_urg(struct sock *sk, long timeo,
887 			struct msghdr *msg, int len, int flags,
888 			int *addr_len)
889 {
890 	struct tcp_sock *tp = tcp_sk(sk);
891 
892 	/* No URG data to read. */
893 	if (sock_flag(sk, SOCK_URGINLINE) || !tp->urg_data ||
894 	    tp->urg_data == TCP_URG_READ)
895 		return -EINVAL;	/* Yes this is right ! */
896 
897 	if (sk->sk_state == TCP_CLOSE && !sock_flag(sk, SOCK_DONE))
898 		return -ENOTCONN;
899 
900 	if (tp->urg_data & TCP_URG_VALID) {
901 		int err = 0;
902 		char c = tp->urg_data;
903 
904 		if (!(flags & MSG_PEEK))
905 			tp->urg_data = TCP_URG_READ;
906 
907 		/* Read urgent data. */
908 		msg->msg_flags |= MSG_OOB;
909 
910 		if (len > 0) {
911 			if (!(flags & MSG_TRUNC))
912 				err = memcpy_toiovec(msg->msg_iov, &c, 1);
913 			len = 1;
914 		} else
915 			msg->msg_flags |= MSG_TRUNC;
916 
917 		return err ? -EFAULT : len;
918 	}
919 
920 	if (sk->sk_state == TCP_CLOSE || (sk->sk_shutdown & RCV_SHUTDOWN))
921 		return 0;
922 
923 	/* Fixed the recv(..., MSG_OOB) behaviour.  BSD docs and
924 	 * the available implementations agree in this case:
925 	 * this call should never block, independent of the
926 	 * blocking state of the socket.
927 	 * Mike <pall@rz.uni-karlsruhe.de>
928 	 */
929 	return -EAGAIN;
930 }
931 
932 /* Clean up the receive buffer for full frames taken by the user,
933  * then send an ACK if necessary.  COPIED is the number of bytes
934  * tcp_recvmsg has given to the user so far, it speeds up the
935  * calculation of whether or not we must ACK for the sake of
936  * a window update.
937  */
938 void tcp_cleanup_rbuf(struct sock *sk, int copied)
939 {
940 	struct tcp_sock *tp = tcp_sk(sk);
941 	int time_to_ack = 0;
942 
943 #if TCP_DEBUG
944 	struct sk_buff *skb = skb_peek(&sk->sk_receive_queue);
945 
946 	BUG_TRAP(!skb || before(tp->copied_seq, TCP_SKB_CB(skb)->end_seq));
947 #endif
948 
949 	if (inet_csk_ack_scheduled(sk)) {
950 		const struct inet_connection_sock *icsk = inet_csk(sk);
951 		   /* Delayed ACKs frequently hit locked sockets during bulk
952 		    * receive. */
953 		if (icsk->icsk_ack.blocked ||
954 		    /* Once-per-two-segments ACK was not sent by tcp_input.c */
955 		    tp->rcv_nxt - tp->rcv_wup > icsk->icsk_ack.rcv_mss ||
956 		    /*
957 		     * If this read emptied read buffer, we send ACK, if
958 		     * connection is not bidirectional, user drained
959 		     * receive buffer and there was a small segment
960 		     * in queue.
961 		     */
962 		    (copied > 0 &&
963 		     ((icsk->icsk_ack.pending & ICSK_ACK_PUSHED2) ||
964 		      ((icsk->icsk_ack.pending & ICSK_ACK_PUSHED) &&
965 		       !icsk->icsk_ack.pingpong)) &&
966 		      !atomic_read(&sk->sk_rmem_alloc)))
967 			time_to_ack = 1;
968 	}
969 
970 	/* We send an ACK if we can now advertise a non-zero window
971 	 * which has been raised "significantly".
972 	 *
973 	 * Even if window raised up to infinity, do not send window open ACK
974 	 * in states, where we will not receive more. It is useless.
975 	 */
976 	if (copied > 0 && !time_to_ack && !(sk->sk_shutdown & RCV_SHUTDOWN)) {
977 		__u32 rcv_window_now = tcp_receive_window(tp);
978 
979 		/* Optimize, __tcp_select_window() is not cheap. */
980 		if (2*rcv_window_now <= tp->window_clamp) {
981 			__u32 new_window = __tcp_select_window(sk);
982 
983 			/* Send ACK now, if this read freed lots of space
984 			 * in our buffer. Certainly, new_window is new window.
985 			 * We can advertise it now, if it is not less than current one.
986 			 * "Lots" means "at least twice" here.
987 			 */
988 			if (new_window && new_window >= 2 * rcv_window_now)
989 				time_to_ack = 1;
990 		}
991 	}
992 	if (time_to_ack)
993 		tcp_send_ack(sk);
994 }
995 
996 static void tcp_prequeue_process(struct sock *sk)
997 {
998 	struct sk_buff *skb;
999 	struct tcp_sock *tp = tcp_sk(sk);
1000 
1001 	NET_INC_STATS_USER(LINUX_MIB_TCPPREQUEUED);
1002 
1003 	/* RX process wants to run with disabled BHs, though it is not
1004 	 * necessary */
1005 	local_bh_disable();
1006 	while ((skb = __skb_dequeue(&tp->ucopy.prequeue)) != NULL)
1007 		sk->sk_backlog_rcv(sk, skb);
1008 	local_bh_enable();
1009 
1010 	/* Clear memory counter. */
1011 	tp->ucopy.memory = 0;
1012 }
1013 
1014 static inline struct sk_buff *tcp_recv_skb(struct sock *sk, u32 seq, u32 *off)
1015 {
1016 	struct sk_buff *skb;
1017 	u32 offset;
1018 
1019 	skb_queue_walk(&sk->sk_receive_queue, skb) {
1020 		offset = seq - TCP_SKB_CB(skb)->seq;
1021 		if (tcp_hdr(skb)->syn)
1022 			offset--;
1023 		if (offset < skb->len || tcp_hdr(skb)->fin) {
1024 			*off = offset;
1025 			return skb;
1026 		}
1027 	}
1028 	return NULL;
1029 }
1030 
1031 /*
1032  * This routine provides an alternative to tcp_recvmsg() for routines
1033  * that would like to handle copying from skbuffs directly in 'sendfile'
1034  * fashion.
1035  * Note:
1036  *	- It is assumed that the socket was locked by the caller.
1037  *	- The routine does not block.
1038  *	- At present, there is no support for reading OOB data
1039  *	  or for 'peeking' the socket using this routine
1040  *	  (although both would be easy to implement).
1041  */
1042 int tcp_read_sock(struct sock *sk, read_descriptor_t *desc,
1043 		  sk_read_actor_t recv_actor)
1044 {
1045 	struct sk_buff *skb;
1046 	struct tcp_sock *tp = tcp_sk(sk);
1047 	u32 seq = tp->copied_seq;
1048 	u32 offset;
1049 	int copied = 0;
1050 
1051 	if (sk->sk_state == TCP_LISTEN)
1052 		return -ENOTCONN;
1053 	while ((skb = tcp_recv_skb(sk, seq, &offset)) != NULL) {
1054 		if (offset < skb->len) {
1055 			size_t used, len;
1056 
1057 			len = skb->len - offset;
1058 			/* Stop reading if we hit a patch of urgent data */
1059 			if (tp->urg_data) {
1060 				u32 urg_offset = tp->urg_seq - seq;
1061 				if (urg_offset < len)
1062 					len = urg_offset;
1063 				if (!len)
1064 					break;
1065 			}
1066 			used = recv_actor(desc, skb, offset, len);
1067 			if (used <= len) {
1068 				seq += used;
1069 				copied += used;
1070 				offset += used;
1071 			}
1072 			if (offset != skb->len)
1073 				break;
1074 		}
1075 		if (tcp_hdr(skb)->fin) {
1076 			sk_eat_skb(sk, skb, 0);
1077 			++seq;
1078 			break;
1079 		}
1080 		sk_eat_skb(sk, skb, 0);
1081 		if (!desc->count)
1082 			break;
1083 	}
1084 	tp->copied_seq = seq;
1085 
1086 	tcp_rcv_space_adjust(sk);
1087 
1088 	/* Clean up data we have read: This will do ACK frames. */
1089 	if (copied)
1090 		tcp_cleanup_rbuf(sk, copied);
1091 	return copied;
1092 }
1093 
1094 /*
1095  *	This routine copies from a sock struct into the user buffer.
1096  *
1097  *	Technical note: in 2.3 we work on _locked_ socket, so that
1098  *	tricks with *seq access order and skb->users are not required.
1099  *	Probably, code can be easily improved even more.
1100  */
1101 
1102 int tcp_recvmsg(struct kiocb *iocb, struct sock *sk, struct msghdr *msg,
1103 		size_t len, int nonblock, int flags, int *addr_len)
1104 {
1105 	struct tcp_sock *tp = tcp_sk(sk);
1106 	int copied = 0;
1107 	u32 peek_seq;
1108 	u32 *seq;
1109 	unsigned long used;
1110 	int err;
1111 	int target;		/* Read at least this many bytes */
1112 	long timeo;
1113 	struct task_struct *user_recv = NULL;
1114 	int copied_early = 0;
1115 
1116 	lock_sock(sk);
1117 
1118 	TCP_CHECK_TIMER(sk);
1119 
1120 	err = -ENOTCONN;
1121 	if (sk->sk_state == TCP_LISTEN)
1122 		goto out;
1123 
1124 	timeo = sock_rcvtimeo(sk, nonblock);
1125 
1126 	/* Urgent data needs to be handled specially. */
1127 	if (flags & MSG_OOB)
1128 		goto recv_urg;
1129 
1130 	seq = &tp->copied_seq;
1131 	if (flags & MSG_PEEK) {
1132 		peek_seq = tp->copied_seq;
1133 		seq = &peek_seq;
1134 	}
1135 
1136 	target = sock_rcvlowat(sk, flags & MSG_WAITALL, len);
1137 
1138 #ifdef CONFIG_NET_DMA
1139 	tp->ucopy.dma_chan = NULL;
1140 	preempt_disable();
1141 	if ((len > sysctl_tcp_dma_copybreak) && !(flags & MSG_PEEK) &&
1142 	    !sysctl_tcp_low_latency && __get_cpu_var(softnet_data).net_dma) {
1143 		preempt_enable_no_resched();
1144 		tp->ucopy.pinned_list = dma_pin_iovec_pages(msg->msg_iov, len);
1145 	} else
1146 		preempt_enable_no_resched();
1147 #endif
1148 
1149 	do {
1150 		struct sk_buff *skb;
1151 		u32 offset;
1152 
1153 		/* Are we at urgent data? Stop if we have read anything or have SIGURG pending. */
1154 		if (tp->urg_data && tp->urg_seq == *seq) {
1155 			if (copied)
1156 				break;
1157 			if (signal_pending(current)) {
1158 				copied = timeo ? sock_intr_errno(timeo) : -EAGAIN;
1159 				break;
1160 			}
1161 		}
1162 
1163 		/* Next get a buffer. */
1164 
1165 		skb = skb_peek(&sk->sk_receive_queue);
1166 		do {
1167 			if (!skb)
1168 				break;
1169 
1170 			/* Now that we have two receive queues this
1171 			 * shouldn't happen.
1172 			 */
1173 			if (before(*seq, TCP_SKB_CB(skb)->seq)) {
1174 				printk(KERN_INFO "recvmsg bug: copied %X "
1175 				       "seq %X\n", *seq, TCP_SKB_CB(skb)->seq);
1176 				break;
1177 			}
1178 			offset = *seq - TCP_SKB_CB(skb)->seq;
1179 			if (tcp_hdr(skb)->syn)
1180 				offset--;
1181 			if (offset < skb->len)
1182 				goto found_ok_skb;
1183 			if (tcp_hdr(skb)->fin)
1184 				goto found_fin_ok;
1185 			BUG_TRAP(flags & MSG_PEEK);
1186 			skb = skb->next;
1187 		} while (skb != (struct sk_buff *)&sk->sk_receive_queue);
1188 
1189 		/* Well, if we have backlog, try to process it now yet. */
1190 
1191 		if (copied >= target && !sk->sk_backlog.tail)
1192 			break;
1193 
1194 		if (copied) {
1195 			if (sk->sk_err ||
1196 			    sk->sk_state == TCP_CLOSE ||
1197 			    (sk->sk_shutdown & RCV_SHUTDOWN) ||
1198 			    !timeo ||
1199 			    signal_pending(current) ||
1200 			    (flags & MSG_PEEK))
1201 				break;
1202 		} else {
1203 			if (sock_flag(sk, SOCK_DONE))
1204 				break;
1205 
1206 			if (sk->sk_err) {
1207 				copied = sock_error(sk);
1208 				break;
1209 			}
1210 
1211 			if (sk->sk_shutdown & RCV_SHUTDOWN)
1212 				break;
1213 
1214 			if (sk->sk_state == TCP_CLOSE) {
1215 				if (!sock_flag(sk, SOCK_DONE)) {
1216 					/* This occurs when user tries to read
1217 					 * from never connected socket.
1218 					 */
1219 					copied = -ENOTCONN;
1220 					break;
1221 				}
1222 				break;
1223 			}
1224 
1225 			if (!timeo) {
1226 				copied = -EAGAIN;
1227 				break;
1228 			}
1229 
1230 			if (signal_pending(current)) {
1231 				copied = sock_intr_errno(timeo);
1232 				break;
1233 			}
1234 		}
1235 
1236 		tcp_cleanup_rbuf(sk, copied);
1237 
1238 		if (!sysctl_tcp_low_latency && tp->ucopy.task == user_recv) {
1239 			/* Install new reader */
1240 			if (!user_recv && !(flags & (MSG_TRUNC | MSG_PEEK))) {
1241 				user_recv = current;
1242 				tp->ucopy.task = user_recv;
1243 				tp->ucopy.iov = msg->msg_iov;
1244 			}
1245 
1246 			tp->ucopy.len = len;
1247 
1248 			BUG_TRAP(tp->copied_seq == tp->rcv_nxt ||
1249 				 (flags & (MSG_PEEK | MSG_TRUNC)));
1250 
1251 			/* Ugly... If prequeue is not empty, we have to
1252 			 * process it before releasing socket, otherwise
1253 			 * order will be broken at second iteration.
1254 			 * More elegant solution is required!!!
1255 			 *
1256 			 * Look: we have the following (pseudo)queues:
1257 			 *
1258 			 * 1. packets in flight
1259 			 * 2. backlog
1260 			 * 3. prequeue
1261 			 * 4. receive_queue
1262 			 *
1263 			 * Each queue can be processed only if the next ones
1264 			 * are empty. At this point we have empty receive_queue.
1265 			 * But prequeue _can_ be not empty after 2nd iteration,
1266 			 * when we jumped to start of loop because backlog
1267 			 * processing added something to receive_queue.
1268 			 * We cannot release_sock(), because backlog contains
1269 			 * packets arrived _after_ prequeued ones.
1270 			 *
1271 			 * Shortly, algorithm is clear --- to process all
1272 			 * the queues in order. We could make it more directly,
1273 			 * requeueing packets from backlog to prequeue, if
1274 			 * is not empty. It is more elegant, but eats cycles,
1275 			 * unfortunately.
1276 			 */
1277 			if (!skb_queue_empty(&tp->ucopy.prequeue))
1278 				goto do_prequeue;
1279 
1280 			/* __ Set realtime policy in scheduler __ */
1281 		}
1282 
1283 		if (copied >= target) {
1284 			/* Do not sleep, just process backlog. */
1285 			release_sock(sk);
1286 			lock_sock(sk);
1287 		} else
1288 			sk_wait_data(sk, &timeo);
1289 
1290 #ifdef CONFIG_NET_DMA
1291 		tp->ucopy.wakeup = 0;
1292 #endif
1293 
1294 		if (user_recv) {
1295 			int chunk;
1296 
1297 			/* __ Restore normal policy in scheduler __ */
1298 
1299 			if ((chunk = len - tp->ucopy.len) != 0) {
1300 				NET_ADD_STATS_USER(LINUX_MIB_TCPDIRECTCOPYFROMBACKLOG, chunk);
1301 				len -= chunk;
1302 				copied += chunk;
1303 			}
1304 
1305 			if (tp->rcv_nxt == tp->copied_seq &&
1306 			    !skb_queue_empty(&tp->ucopy.prequeue)) {
1307 do_prequeue:
1308 				tcp_prequeue_process(sk);
1309 
1310 				if ((chunk = len - tp->ucopy.len) != 0) {
1311 					NET_ADD_STATS_USER(LINUX_MIB_TCPDIRECTCOPYFROMPREQUEUE, chunk);
1312 					len -= chunk;
1313 					copied += chunk;
1314 				}
1315 			}
1316 		}
1317 		if ((flags & MSG_PEEK) && peek_seq != tp->copied_seq) {
1318 			if (net_ratelimit())
1319 				printk(KERN_DEBUG "TCP(%s:%d): Application bug, race in MSG_PEEK.\n",
1320 				       current->comm, current->pid);
1321 			peek_seq = tp->copied_seq;
1322 		}
1323 		continue;
1324 
1325 	found_ok_skb:
1326 		/* Ok so how much can we use? */
1327 		used = skb->len - offset;
1328 		if (len < used)
1329 			used = len;
1330 
1331 		/* Do we have urgent data here? */
1332 		if (tp->urg_data) {
1333 			u32 urg_offset = tp->urg_seq - *seq;
1334 			if (urg_offset < used) {
1335 				if (!urg_offset) {
1336 					if (!sock_flag(sk, SOCK_URGINLINE)) {
1337 						++*seq;
1338 						offset++;
1339 						used--;
1340 						if (!used)
1341 							goto skip_copy;
1342 					}
1343 				} else
1344 					used = urg_offset;
1345 			}
1346 		}
1347 
1348 		if (!(flags & MSG_TRUNC)) {
1349 #ifdef CONFIG_NET_DMA
1350 			if (!tp->ucopy.dma_chan && tp->ucopy.pinned_list)
1351 				tp->ucopy.dma_chan = get_softnet_dma();
1352 
1353 			if (tp->ucopy.dma_chan) {
1354 				tp->ucopy.dma_cookie = dma_skb_copy_datagram_iovec(
1355 					tp->ucopy.dma_chan, skb, offset,
1356 					msg->msg_iov, used,
1357 					tp->ucopy.pinned_list);
1358 
1359 				if (tp->ucopy.dma_cookie < 0) {
1360 
1361 					printk(KERN_ALERT "dma_cookie < 0\n");
1362 
1363 					/* Exception. Bailout! */
1364 					if (!copied)
1365 						copied = -EFAULT;
1366 					break;
1367 				}
1368 				if ((offset + used) == skb->len)
1369 					copied_early = 1;
1370 
1371 			} else
1372 #endif
1373 			{
1374 				err = skb_copy_datagram_iovec(skb, offset,
1375 						msg->msg_iov, used);
1376 				if (err) {
1377 					/* Exception. Bailout! */
1378 					if (!copied)
1379 						copied = -EFAULT;
1380 					break;
1381 				}
1382 			}
1383 		}
1384 
1385 		*seq += used;
1386 		copied += used;
1387 		len -= used;
1388 
1389 		tcp_rcv_space_adjust(sk);
1390 
1391 skip_copy:
1392 		if (tp->urg_data && after(tp->copied_seq, tp->urg_seq)) {
1393 			tp->urg_data = 0;
1394 			tcp_fast_path_check(sk);
1395 		}
1396 		if (used + offset < skb->len)
1397 			continue;
1398 
1399 		if (tcp_hdr(skb)->fin)
1400 			goto found_fin_ok;
1401 		if (!(flags & MSG_PEEK)) {
1402 			sk_eat_skb(sk, skb, copied_early);
1403 			copied_early = 0;
1404 		}
1405 		continue;
1406 
1407 	found_fin_ok:
1408 		/* Process the FIN. */
1409 		++*seq;
1410 		if (!(flags & MSG_PEEK)) {
1411 			sk_eat_skb(sk, skb, copied_early);
1412 			copied_early = 0;
1413 		}
1414 		break;
1415 	} while (len > 0);
1416 
1417 	if (user_recv) {
1418 		if (!skb_queue_empty(&tp->ucopy.prequeue)) {
1419 			int chunk;
1420 
1421 			tp->ucopy.len = copied > 0 ? len : 0;
1422 
1423 			tcp_prequeue_process(sk);
1424 
1425 			if (copied > 0 && (chunk = len - tp->ucopy.len) != 0) {
1426 				NET_ADD_STATS_USER(LINUX_MIB_TCPDIRECTCOPYFROMPREQUEUE, chunk);
1427 				len -= chunk;
1428 				copied += chunk;
1429 			}
1430 		}
1431 
1432 		tp->ucopy.task = NULL;
1433 		tp->ucopy.len = 0;
1434 	}
1435 
1436 #ifdef CONFIG_NET_DMA
1437 	if (tp->ucopy.dma_chan) {
1438 		struct sk_buff *skb;
1439 		dma_cookie_t done, used;
1440 
1441 		dma_async_memcpy_issue_pending(tp->ucopy.dma_chan);
1442 
1443 		while (dma_async_memcpy_complete(tp->ucopy.dma_chan,
1444 						 tp->ucopy.dma_cookie, &done,
1445 						 &used) == DMA_IN_PROGRESS) {
1446 			/* do partial cleanup of sk_async_wait_queue */
1447 			while ((skb = skb_peek(&sk->sk_async_wait_queue)) &&
1448 			       (dma_async_is_complete(skb->dma_cookie, done,
1449 						      used) == DMA_SUCCESS)) {
1450 				__skb_dequeue(&sk->sk_async_wait_queue);
1451 				kfree_skb(skb);
1452 			}
1453 		}
1454 
1455 		/* Safe to free early-copied skbs now */
1456 		__skb_queue_purge(&sk->sk_async_wait_queue);
1457 		dma_chan_put(tp->ucopy.dma_chan);
1458 		tp->ucopy.dma_chan = NULL;
1459 	}
1460 	if (tp->ucopy.pinned_list) {
1461 		dma_unpin_iovec_pages(tp->ucopy.pinned_list);
1462 		tp->ucopy.pinned_list = NULL;
1463 	}
1464 #endif
1465 
1466 	/* According to UNIX98, msg_name/msg_namelen are ignored
1467 	 * on connected socket. I was just happy when found this 8) --ANK
1468 	 */
1469 
1470 	/* Clean up data we have read: This will do ACK frames. */
1471 	tcp_cleanup_rbuf(sk, copied);
1472 
1473 	TCP_CHECK_TIMER(sk);
1474 	release_sock(sk);
1475 	return copied;
1476 
1477 out:
1478 	TCP_CHECK_TIMER(sk);
1479 	release_sock(sk);
1480 	return err;
1481 
1482 recv_urg:
1483 	err = tcp_recv_urg(sk, timeo, msg, len, flags, addr_len);
1484 	goto out;
1485 }
1486 
1487 /*
1488  *	State processing on a close. This implements the state shift for
1489  *	sending our FIN frame. Note that we only send a FIN for some
1490  *	states. A shutdown() may have already sent the FIN, or we may be
1491  *	closed.
1492  */
1493 
1494 static const unsigned char new_state[16] = {
1495   /* current state:        new state:      action:	*/
1496   /* (Invalid)		*/ TCP_CLOSE,
1497   /* TCP_ESTABLISHED	*/ TCP_FIN_WAIT1 | TCP_ACTION_FIN,
1498   /* TCP_SYN_SENT	*/ TCP_CLOSE,
1499   /* TCP_SYN_RECV	*/ TCP_FIN_WAIT1 | TCP_ACTION_FIN,
1500   /* TCP_FIN_WAIT1	*/ TCP_FIN_WAIT1,
1501   /* TCP_FIN_WAIT2	*/ TCP_FIN_WAIT2,
1502   /* TCP_TIME_WAIT	*/ TCP_CLOSE,
1503   /* TCP_CLOSE		*/ TCP_CLOSE,
1504   /* TCP_CLOSE_WAIT	*/ TCP_LAST_ACK  | TCP_ACTION_FIN,
1505   /* TCP_LAST_ACK	*/ TCP_LAST_ACK,
1506   /* TCP_LISTEN		*/ TCP_CLOSE,
1507   /* TCP_CLOSING	*/ TCP_CLOSING,
1508 };
1509 
1510 static int tcp_close_state(struct sock *sk)
1511 {
1512 	int next = (int)new_state[sk->sk_state];
1513 	int ns = next & TCP_STATE_MASK;
1514 
1515 	tcp_set_state(sk, ns);
1516 
1517 	return next & TCP_ACTION_FIN;
1518 }
1519 
1520 /*
1521  *	Shutdown the sending side of a connection. Much like close except
1522  *	that we don't receive shut down or set_sock_flag(sk, SOCK_DEAD).
1523  */
1524 
1525 void tcp_shutdown(struct sock *sk, int how)
1526 {
1527 	/*	We need to grab some memory, and put together a FIN,
1528 	 *	and then put it into the queue to be sent.
1529 	 *		Tim MacKenzie(tym@dibbler.cs.monash.edu.au) 4 Dec '92.
1530 	 */
1531 	if (!(how & SEND_SHUTDOWN))
1532 		return;
1533 
1534 	/* If we've already sent a FIN, or it's a closed state, skip this. */
1535 	if ((1 << sk->sk_state) &
1536 	    (TCPF_ESTABLISHED | TCPF_SYN_SENT |
1537 	     TCPF_SYN_RECV | TCPF_CLOSE_WAIT)) {
1538 		/* Clear out any half completed packets.  FIN if needed. */
1539 		if (tcp_close_state(sk))
1540 			tcp_send_fin(sk);
1541 	}
1542 }
1543 
1544 void tcp_close(struct sock *sk, long timeout)
1545 {
1546 	struct sk_buff *skb;
1547 	int data_was_unread = 0;
1548 	int state;
1549 
1550 	lock_sock(sk);
1551 	sk->sk_shutdown = SHUTDOWN_MASK;
1552 
1553 	if (sk->sk_state == TCP_LISTEN) {
1554 		tcp_set_state(sk, TCP_CLOSE);
1555 
1556 		/* Special case. */
1557 		inet_csk_listen_stop(sk);
1558 
1559 		goto adjudge_to_death;
1560 	}
1561 
1562 	/*  We need to flush the recv. buffs.  We do this only on the
1563 	 *  descriptor close, not protocol-sourced closes, because the
1564 	 *  reader process may not have drained the data yet!
1565 	 */
1566 	while ((skb = __skb_dequeue(&sk->sk_receive_queue)) != NULL) {
1567 		u32 len = TCP_SKB_CB(skb)->end_seq - TCP_SKB_CB(skb)->seq -
1568 			  tcp_hdr(skb)->fin;
1569 		data_was_unread += len;
1570 		__kfree_skb(skb);
1571 	}
1572 
1573 	sk_stream_mem_reclaim(sk);
1574 
1575 	/* As outlined in RFC 2525, section 2.17, we send a RST here because
1576 	 * data was lost. To witness the awful effects of the old behavior of
1577 	 * always doing a FIN, run an older 2.1.x kernel or 2.0.x, start a bulk
1578 	 * GET in an FTP client, suspend the process, wait for the client to
1579 	 * advertise a zero window, then kill -9 the FTP client, wheee...
1580 	 * Note: timeout is always zero in such a case.
1581 	 */
1582 	if (data_was_unread) {
1583 		/* Unread data was tossed, zap the connection. */
1584 		NET_INC_STATS_USER(LINUX_MIB_TCPABORTONCLOSE);
1585 		tcp_set_state(sk, TCP_CLOSE);
1586 		tcp_send_active_reset(sk, GFP_KERNEL);
1587 	} else if (sock_flag(sk, SOCK_LINGER) && !sk->sk_lingertime) {
1588 		/* Check zero linger _after_ checking for unread data. */
1589 		sk->sk_prot->disconnect(sk, 0);
1590 		NET_INC_STATS_USER(LINUX_MIB_TCPABORTONDATA);
1591 	} else if (tcp_close_state(sk)) {
1592 		/* We FIN if the application ate all the data before
1593 		 * zapping the connection.
1594 		 */
1595 
1596 		/* RED-PEN. Formally speaking, we have broken TCP state
1597 		 * machine. State transitions:
1598 		 *
1599 		 * TCP_ESTABLISHED -> TCP_FIN_WAIT1
1600 		 * TCP_SYN_RECV	-> TCP_FIN_WAIT1 (forget it, it's impossible)
1601 		 * TCP_CLOSE_WAIT -> TCP_LAST_ACK
1602 		 *
1603 		 * are legal only when FIN has been sent (i.e. in window),
1604 		 * rather than queued out of window. Purists blame.
1605 		 *
1606 		 * F.e. "RFC state" is ESTABLISHED,
1607 		 * if Linux state is FIN-WAIT-1, but FIN is still not sent.
1608 		 *
1609 		 * The visible declinations are that sometimes
1610 		 * we enter time-wait state, when it is not required really
1611 		 * (harmless), do not send active resets, when they are
1612 		 * required by specs (TCP_ESTABLISHED, TCP_CLOSE_WAIT, when
1613 		 * they look as CLOSING or LAST_ACK for Linux)
1614 		 * Probably, I missed some more holelets.
1615 		 * 						--ANK
1616 		 */
1617 		tcp_send_fin(sk);
1618 	}
1619 
1620 	sk_stream_wait_close(sk, timeout);
1621 
1622 adjudge_to_death:
1623 	state = sk->sk_state;
1624 	sock_hold(sk);
1625 	sock_orphan(sk);
1626 	atomic_inc(sk->sk_prot->orphan_count);
1627 
1628 	/* It is the last release_sock in its life. It will remove backlog. */
1629 	release_sock(sk);
1630 
1631 
1632 	/* Now socket is owned by kernel and we acquire BH lock
1633 	   to finish close. No need to check for user refs.
1634 	 */
1635 	local_bh_disable();
1636 	bh_lock_sock(sk);
1637 	BUG_TRAP(!sock_owned_by_user(sk));
1638 
1639 	/* Have we already been destroyed by a softirq or backlog? */
1640 	if (state != TCP_CLOSE && sk->sk_state == TCP_CLOSE)
1641 		goto out;
1642 
1643 	/*	This is a (useful) BSD violating of the RFC. There is a
1644 	 *	problem with TCP as specified in that the other end could
1645 	 *	keep a socket open forever with no application left this end.
1646 	 *	We use a 3 minute timeout (about the same as BSD) then kill
1647 	 *	our end. If they send after that then tough - BUT: long enough
1648 	 *	that we won't make the old 4*rto = almost no time - whoops
1649 	 *	reset mistake.
1650 	 *
1651 	 *	Nope, it was not mistake. It is really desired behaviour
1652 	 *	f.e. on http servers, when such sockets are useless, but
1653 	 *	consume significant resources. Let's do it with special
1654 	 *	linger2	option.					--ANK
1655 	 */
1656 
1657 	if (sk->sk_state == TCP_FIN_WAIT2) {
1658 		struct tcp_sock *tp = tcp_sk(sk);
1659 		if (tp->linger2 < 0) {
1660 			tcp_set_state(sk, TCP_CLOSE);
1661 			tcp_send_active_reset(sk, GFP_ATOMIC);
1662 			NET_INC_STATS_BH(LINUX_MIB_TCPABORTONLINGER);
1663 		} else {
1664 			const int tmo = tcp_fin_time(sk);
1665 
1666 			if (tmo > TCP_TIMEWAIT_LEN) {
1667 				inet_csk_reset_keepalive_timer(sk,
1668 						tmo - TCP_TIMEWAIT_LEN);
1669 			} else {
1670 				tcp_time_wait(sk, TCP_FIN_WAIT2, tmo);
1671 				goto out;
1672 			}
1673 		}
1674 	}
1675 	if (sk->sk_state != TCP_CLOSE) {
1676 		sk_stream_mem_reclaim(sk);
1677 		if (atomic_read(sk->sk_prot->orphan_count) > sysctl_tcp_max_orphans ||
1678 		    (sk->sk_wmem_queued > SOCK_MIN_SNDBUF &&
1679 		     atomic_read(&tcp_memory_allocated) > sysctl_tcp_mem[2])) {
1680 			if (net_ratelimit())
1681 				printk(KERN_INFO "TCP: too many of orphaned "
1682 				       "sockets\n");
1683 			tcp_set_state(sk, TCP_CLOSE);
1684 			tcp_send_active_reset(sk, GFP_ATOMIC);
1685 			NET_INC_STATS_BH(LINUX_MIB_TCPABORTONMEMORY);
1686 		}
1687 	}
1688 
1689 	if (sk->sk_state == TCP_CLOSE)
1690 		inet_csk_destroy_sock(sk);
1691 	/* Otherwise, socket is reprieved until protocol close. */
1692 
1693 out:
1694 	bh_unlock_sock(sk);
1695 	local_bh_enable();
1696 	sock_put(sk);
1697 }
1698 
1699 /* These states need RST on ABORT according to RFC793 */
1700 
1701 static inline int tcp_need_reset(int state)
1702 {
1703 	return (1 << state) &
1704 	       (TCPF_ESTABLISHED | TCPF_CLOSE_WAIT | TCPF_FIN_WAIT1 |
1705 		TCPF_FIN_WAIT2 | TCPF_SYN_RECV);
1706 }
1707 
1708 int tcp_disconnect(struct sock *sk, int flags)
1709 {
1710 	struct inet_sock *inet = inet_sk(sk);
1711 	struct inet_connection_sock *icsk = inet_csk(sk);
1712 	struct tcp_sock *tp = tcp_sk(sk);
1713 	int err = 0;
1714 	int old_state = sk->sk_state;
1715 
1716 	if (old_state != TCP_CLOSE)
1717 		tcp_set_state(sk, TCP_CLOSE);
1718 
1719 	/* ABORT function of RFC793 */
1720 	if (old_state == TCP_LISTEN) {
1721 		inet_csk_listen_stop(sk);
1722 	} else if (tcp_need_reset(old_state) ||
1723 		   (tp->snd_nxt != tp->write_seq &&
1724 		    (1 << old_state) & (TCPF_CLOSING | TCPF_LAST_ACK))) {
1725 		/* The last check adjusts for discrepancy of Linux wrt. RFC
1726 		 * states
1727 		 */
1728 		tcp_send_active_reset(sk, gfp_any());
1729 		sk->sk_err = ECONNRESET;
1730 	} else if (old_state == TCP_SYN_SENT)
1731 		sk->sk_err = ECONNRESET;
1732 
1733 	tcp_clear_xmit_timers(sk);
1734 	__skb_queue_purge(&sk->sk_receive_queue);
1735 	tcp_write_queue_purge(sk);
1736 	__skb_queue_purge(&tp->out_of_order_queue);
1737 #ifdef CONFIG_NET_DMA
1738 	__skb_queue_purge(&sk->sk_async_wait_queue);
1739 #endif
1740 
1741 	inet->dport = 0;
1742 
1743 	if (!(sk->sk_userlocks & SOCK_BINDADDR_LOCK))
1744 		inet_reset_saddr(sk);
1745 
1746 	sk->sk_shutdown = 0;
1747 	sock_reset_flag(sk, SOCK_DONE);
1748 	tp->srtt = 0;
1749 	if ((tp->write_seq += tp->max_window + 2) == 0)
1750 		tp->write_seq = 1;
1751 	icsk->icsk_backoff = 0;
1752 	tp->snd_cwnd = 2;
1753 	icsk->icsk_probes_out = 0;
1754 	tp->packets_out = 0;
1755 	tp->snd_ssthresh = 0x7fffffff;
1756 	tp->snd_cwnd_cnt = 0;
1757 	tp->bytes_acked = 0;
1758 	tcp_set_ca_state(sk, TCP_CA_Open);
1759 	tcp_clear_retrans(tp);
1760 	inet_csk_delack_init(sk);
1761 	tcp_init_send_head(sk);
1762 	memset(&tp->rx_opt, 0, sizeof(tp->rx_opt));
1763 	__sk_dst_reset(sk);
1764 
1765 	BUG_TRAP(!inet->num || icsk->icsk_bind_hash);
1766 
1767 	sk->sk_error_report(sk);
1768 	return err;
1769 }
1770 
1771 /*
1772  *	Socket option code for TCP.
1773  */
1774 static int do_tcp_setsockopt(struct sock *sk, int level,
1775 		int optname, char __user *optval, int optlen)
1776 {
1777 	struct tcp_sock *tp = tcp_sk(sk);
1778 	struct inet_connection_sock *icsk = inet_csk(sk);
1779 	int val;
1780 	int err = 0;
1781 
1782 	/* This is a string value all the others are int's */
1783 	if (optname == TCP_CONGESTION) {
1784 		char name[TCP_CA_NAME_MAX];
1785 
1786 		if (optlen < 1)
1787 			return -EINVAL;
1788 
1789 		val = strncpy_from_user(name, optval,
1790 					min(TCP_CA_NAME_MAX-1, optlen));
1791 		if (val < 0)
1792 			return -EFAULT;
1793 		name[val] = 0;
1794 
1795 		lock_sock(sk);
1796 		err = tcp_set_congestion_control(sk, name);
1797 		release_sock(sk);
1798 		return err;
1799 	}
1800 
1801 	if (optlen < sizeof(int))
1802 		return -EINVAL;
1803 
1804 	if (get_user(val, (int __user *)optval))
1805 		return -EFAULT;
1806 
1807 	lock_sock(sk);
1808 
1809 	switch (optname) {
1810 	case TCP_MAXSEG:
1811 		/* Values greater than interface MTU won't take effect. However
1812 		 * at the point when this call is done we typically don't yet
1813 		 * know which interface is going to be used */
1814 		if (val < 8 || val > MAX_TCP_WINDOW) {
1815 			err = -EINVAL;
1816 			break;
1817 		}
1818 		tp->rx_opt.user_mss = val;
1819 		break;
1820 
1821 	case TCP_NODELAY:
1822 		if (val) {
1823 			/* TCP_NODELAY is weaker than TCP_CORK, so that
1824 			 * this option on corked socket is remembered, but
1825 			 * it is not activated until cork is cleared.
1826 			 *
1827 			 * However, when TCP_NODELAY is set we make
1828 			 * an explicit push, which overrides even TCP_CORK
1829 			 * for currently queued segments.
1830 			 */
1831 			tp->nonagle |= TCP_NAGLE_OFF|TCP_NAGLE_PUSH;
1832 			tcp_push_pending_frames(sk);
1833 		} else {
1834 			tp->nonagle &= ~TCP_NAGLE_OFF;
1835 		}
1836 		break;
1837 
1838 	case TCP_CORK:
1839 		/* When set indicates to always queue non-full frames.
1840 		 * Later the user clears this option and we transmit
1841 		 * any pending partial frames in the queue.  This is
1842 		 * meant to be used alongside sendfile() to get properly
1843 		 * filled frames when the user (for example) must write
1844 		 * out headers with a write() call first and then use
1845 		 * sendfile to send out the data parts.
1846 		 *
1847 		 * TCP_CORK can be set together with TCP_NODELAY and it is
1848 		 * stronger than TCP_NODELAY.
1849 		 */
1850 		if (val) {
1851 			tp->nonagle |= TCP_NAGLE_CORK;
1852 		} else {
1853 			tp->nonagle &= ~TCP_NAGLE_CORK;
1854 			if (tp->nonagle&TCP_NAGLE_OFF)
1855 				tp->nonagle |= TCP_NAGLE_PUSH;
1856 			tcp_push_pending_frames(sk);
1857 		}
1858 		break;
1859 
1860 	case TCP_KEEPIDLE:
1861 		if (val < 1 || val > MAX_TCP_KEEPIDLE)
1862 			err = -EINVAL;
1863 		else {
1864 			tp->keepalive_time = val * HZ;
1865 			if (sock_flag(sk, SOCK_KEEPOPEN) &&
1866 			    !((1 << sk->sk_state) &
1867 			      (TCPF_CLOSE | TCPF_LISTEN))) {
1868 				__u32 elapsed = tcp_time_stamp - tp->rcv_tstamp;
1869 				if (tp->keepalive_time > elapsed)
1870 					elapsed = tp->keepalive_time - elapsed;
1871 				else
1872 					elapsed = 0;
1873 				inet_csk_reset_keepalive_timer(sk, elapsed);
1874 			}
1875 		}
1876 		break;
1877 	case TCP_KEEPINTVL:
1878 		if (val < 1 || val > MAX_TCP_KEEPINTVL)
1879 			err = -EINVAL;
1880 		else
1881 			tp->keepalive_intvl = val * HZ;
1882 		break;
1883 	case TCP_KEEPCNT:
1884 		if (val < 1 || val > MAX_TCP_KEEPCNT)
1885 			err = -EINVAL;
1886 		else
1887 			tp->keepalive_probes = val;
1888 		break;
1889 	case TCP_SYNCNT:
1890 		if (val < 1 || val > MAX_TCP_SYNCNT)
1891 			err = -EINVAL;
1892 		else
1893 			icsk->icsk_syn_retries = val;
1894 		break;
1895 
1896 	case TCP_LINGER2:
1897 		if (val < 0)
1898 			tp->linger2 = -1;
1899 		else if (val > sysctl_tcp_fin_timeout / HZ)
1900 			tp->linger2 = 0;
1901 		else
1902 			tp->linger2 = val * HZ;
1903 		break;
1904 
1905 	case TCP_DEFER_ACCEPT:
1906 		icsk->icsk_accept_queue.rskq_defer_accept = 0;
1907 		if (val > 0) {
1908 			/* Translate value in seconds to number of
1909 			 * retransmits */
1910 			while (icsk->icsk_accept_queue.rskq_defer_accept < 32 &&
1911 			       val > ((TCP_TIMEOUT_INIT / HZ) <<
1912 				       icsk->icsk_accept_queue.rskq_defer_accept))
1913 				icsk->icsk_accept_queue.rskq_defer_accept++;
1914 			icsk->icsk_accept_queue.rskq_defer_accept++;
1915 		}
1916 		break;
1917 
1918 	case TCP_WINDOW_CLAMP:
1919 		if (!val) {
1920 			if (sk->sk_state != TCP_CLOSE) {
1921 				err = -EINVAL;
1922 				break;
1923 			}
1924 			tp->window_clamp = 0;
1925 		} else
1926 			tp->window_clamp = val < SOCK_MIN_RCVBUF / 2 ?
1927 						SOCK_MIN_RCVBUF / 2 : val;
1928 		break;
1929 
1930 	case TCP_QUICKACK:
1931 		if (!val) {
1932 			icsk->icsk_ack.pingpong = 1;
1933 		} else {
1934 			icsk->icsk_ack.pingpong = 0;
1935 			if ((1 << sk->sk_state) &
1936 			    (TCPF_ESTABLISHED | TCPF_CLOSE_WAIT) &&
1937 			    inet_csk_ack_scheduled(sk)) {
1938 				icsk->icsk_ack.pending |= ICSK_ACK_PUSHED;
1939 				tcp_cleanup_rbuf(sk, 1);
1940 				if (!(val & 1))
1941 					icsk->icsk_ack.pingpong = 1;
1942 			}
1943 		}
1944 		break;
1945 
1946 #ifdef CONFIG_TCP_MD5SIG
1947 	case TCP_MD5SIG:
1948 		/* Read the IP->Key mappings from userspace */
1949 		err = tp->af_specific->md5_parse(sk, optval, optlen);
1950 		break;
1951 #endif
1952 
1953 	default:
1954 		err = -ENOPROTOOPT;
1955 		break;
1956 	}
1957 
1958 	release_sock(sk);
1959 	return err;
1960 }
1961 
1962 int tcp_setsockopt(struct sock *sk, int level, int optname, char __user *optval,
1963 		   int optlen)
1964 {
1965 	struct inet_connection_sock *icsk = inet_csk(sk);
1966 
1967 	if (level != SOL_TCP)
1968 		return icsk->icsk_af_ops->setsockopt(sk, level, optname,
1969 						     optval, optlen);
1970 	return do_tcp_setsockopt(sk, level, optname, optval, optlen);
1971 }
1972 
1973 #ifdef CONFIG_COMPAT
1974 int compat_tcp_setsockopt(struct sock *sk, int level, int optname,
1975 			  char __user *optval, int optlen)
1976 {
1977 	if (level != SOL_TCP)
1978 		return inet_csk_compat_setsockopt(sk, level, optname,
1979 						  optval, optlen);
1980 	return do_tcp_setsockopt(sk, level, optname, optval, optlen);
1981 }
1982 
1983 EXPORT_SYMBOL(compat_tcp_setsockopt);
1984 #endif
1985 
1986 /* Return information about state of tcp endpoint in API format. */
1987 void tcp_get_info(struct sock *sk, struct tcp_info *info)
1988 {
1989 	struct tcp_sock *tp = tcp_sk(sk);
1990 	const struct inet_connection_sock *icsk = inet_csk(sk);
1991 	u32 now = tcp_time_stamp;
1992 
1993 	memset(info, 0, sizeof(*info));
1994 
1995 	info->tcpi_state = sk->sk_state;
1996 	info->tcpi_ca_state = icsk->icsk_ca_state;
1997 	info->tcpi_retransmits = icsk->icsk_retransmits;
1998 	info->tcpi_probes = icsk->icsk_probes_out;
1999 	info->tcpi_backoff = icsk->icsk_backoff;
2000 
2001 	if (tp->rx_opt.tstamp_ok)
2002 		info->tcpi_options |= TCPI_OPT_TIMESTAMPS;
2003 	if (tp->rx_opt.sack_ok)
2004 		info->tcpi_options |= TCPI_OPT_SACK;
2005 	if (tp->rx_opt.wscale_ok) {
2006 		info->tcpi_options |= TCPI_OPT_WSCALE;
2007 		info->tcpi_snd_wscale = tp->rx_opt.snd_wscale;
2008 		info->tcpi_rcv_wscale = tp->rx_opt.rcv_wscale;
2009 	}
2010 
2011 	if (tp->ecn_flags&TCP_ECN_OK)
2012 		info->tcpi_options |= TCPI_OPT_ECN;
2013 
2014 	info->tcpi_rto = jiffies_to_usecs(icsk->icsk_rto);
2015 	info->tcpi_ato = jiffies_to_usecs(icsk->icsk_ack.ato);
2016 	info->tcpi_snd_mss = tp->mss_cache;
2017 	info->tcpi_rcv_mss = icsk->icsk_ack.rcv_mss;
2018 
2019 	info->tcpi_unacked = tp->packets_out;
2020 	info->tcpi_sacked = tp->sacked_out;
2021 	info->tcpi_lost = tp->lost_out;
2022 	info->tcpi_retrans = tp->retrans_out;
2023 	info->tcpi_fackets = tp->fackets_out;
2024 
2025 	info->tcpi_last_data_sent = jiffies_to_msecs(now - tp->lsndtime);
2026 	info->tcpi_last_data_recv = jiffies_to_msecs(now - icsk->icsk_ack.lrcvtime);
2027 	info->tcpi_last_ack_recv = jiffies_to_msecs(now - tp->rcv_tstamp);
2028 
2029 	info->tcpi_pmtu = icsk->icsk_pmtu_cookie;
2030 	info->tcpi_rcv_ssthresh = tp->rcv_ssthresh;
2031 	info->tcpi_rtt = jiffies_to_usecs(tp->srtt)>>3;
2032 	info->tcpi_rttvar = jiffies_to_usecs(tp->mdev)>>2;
2033 	info->tcpi_snd_ssthresh = tp->snd_ssthresh;
2034 	info->tcpi_snd_cwnd = tp->snd_cwnd;
2035 	info->tcpi_advmss = tp->advmss;
2036 	info->tcpi_reordering = tp->reordering;
2037 
2038 	info->tcpi_rcv_rtt = jiffies_to_usecs(tp->rcv_rtt_est.rtt)>>3;
2039 	info->tcpi_rcv_space = tp->rcvq_space.space;
2040 
2041 	info->tcpi_total_retrans = tp->total_retrans;
2042 }
2043 
2044 EXPORT_SYMBOL_GPL(tcp_get_info);
2045 
2046 static int do_tcp_getsockopt(struct sock *sk, int level,
2047 		int optname, char __user *optval, int __user *optlen)
2048 {
2049 	struct inet_connection_sock *icsk = inet_csk(sk);
2050 	struct tcp_sock *tp = tcp_sk(sk);
2051 	int val, len;
2052 
2053 	if (get_user(len, optlen))
2054 		return -EFAULT;
2055 
2056 	len = min_t(unsigned int, len, sizeof(int));
2057 
2058 	if (len < 0)
2059 		return -EINVAL;
2060 
2061 	switch (optname) {
2062 	case TCP_MAXSEG:
2063 		val = tp->mss_cache;
2064 		if (!val && ((1 << sk->sk_state) & (TCPF_CLOSE | TCPF_LISTEN)))
2065 			val = tp->rx_opt.user_mss;
2066 		break;
2067 	case TCP_NODELAY:
2068 		val = !!(tp->nonagle&TCP_NAGLE_OFF);
2069 		break;
2070 	case TCP_CORK:
2071 		val = !!(tp->nonagle&TCP_NAGLE_CORK);
2072 		break;
2073 	case TCP_KEEPIDLE:
2074 		val = (tp->keepalive_time ? : sysctl_tcp_keepalive_time) / HZ;
2075 		break;
2076 	case TCP_KEEPINTVL:
2077 		val = (tp->keepalive_intvl ? : sysctl_tcp_keepalive_intvl) / HZ;
2078 		break;
2079 	case TCP_KEEPCNT:
2080 		val = tp->keepalive_probes ? : sysctl_tcp_keepalive_probes;
2081 		break;
2082 	case TCP_SYNCNT:
2083 		val = icsk->icsk_syn_retries ? : sysctl_tcp_syn_retries;
2084 		break;
2085 	case TCP_LINGER2:
2086 		val = tp->linger2;
2087 		if (val >= 0)
2088 			val = (val ? : sysctl_tcp_fin_timeout) / HZ;
2089 		break;
2090 	case TCP_DEFER_ACCEPT:
2091 		val = !icsk->icsk_accept_queue.rskq_defer_accept ? 0 :
2092 			((TCP_TIMEOUT_INIT / HZ) << (icsk->icsk_accept_queue.rskq_defer_accept - 1));
2093 		break;
2094 	case TCP_WINDOW_CLAMP:
2095 		val = tp->window_clamp;
2096 		break;
2097 	case TCP_INFO: {
2098 		struct tcp_info info;
2099 
2100 		if (get_user(len, optlen))
2101 			return -EFAULT;
2102 
2103 		tcp_get_info(sk, &info);
2104 
2105 		len = min_t(unsigned int, len, sizeof(info));
2106 		if (put_user(len, optlen))
2107 			return -EFAULT;
2108 		if (copy_to_user(optval, &info, len))
2109 			return -EFAULT;
2110 		return 0;
2111 	}
2112 	case TCP_QUICKACK:
2113 		val = !icsk->icsk_ack.pingpong;
2114 		break;
2115 
2116 	case TCP_CONGESTION:
2117 		if (get_user(len, optlen))
2118 			return -EFAULT;
2119 		len = min_t(unsigned int, len, TCP_CA_NAME_MAX);
2120 		if (put_user(len, optlen))
2121 			return -EFAULT;
2122 		if (copy_to_user(optval, icsk->icsk_ca_ops->name, len))
2123 			return -EFAULT;
2124 		return 0;
2125 	default:
2126 		return -ENOPROTOOPT;
2127 	}
2128 
2129 	if (put_user(len, optlen))
2130 		return -EFAULT;
2131 	if (copy_to_user(optval, &val, len))
2132 		return -EFAULT;
2133 	return 0;
2134 }
2135 
2136 int tcp_getsockopt(struct sock *sk, int level, int optname, char __user *optval,
2137 		   int __user *optlen)
2138 {
2139 	struct inet_connection_sock *icsk = inet_csk(sk);
2140 
2141 	if (level != SOL_TCP)
2142 		return icsk->icsk_af_ops->getsockopt(sk, level, optname,
2143 						     optval, optlen);
2144 	return do_tcp_getsockopt(sk, level, optname, optval, optlen);
2145 }
2146 
2147 #ifdef CONFIG_COMPAT
2148 int compat_tcp_getsockopt(struct sock *sk, int level, int optname,
2149 			  char __user *optval, int __user *optlen)
2150 {
2151 	if (level != SOL_TCP)
2152 		return inet_csk_compat_getsockopt(sk, level, optname,
2153 						  optval, optlen);
2154 	return do_tcp_getsockopt(sk, level, optname, optval, optlen);
2155 }
2156 
2157 EXPORT_SYMBOL(compat_tcp_getsockopt);
2158 #endif
2159 
2160 struct sk_buff *tcp_tso_segment(struct sk_buff *skb, int features)
2161 {
2162 	struct sk_buff *segs = ERR_PTR(-EINVAL);
2163 	struct tcphdr *th;
2164 	unsigned thlen;
2165 	unsigned int seq;
2166 	__be32 delta;
2167 	unsigned int oldlen;
2168 	unsigned int len;
2169 
2170 	if (!pskb_may_pull(skb, sizeof(*th)))
2171 		goto out;
2172 
2173 	th = tcp_hdr(skb);
2174 	thlen = th->doff * 4;
2175 	if (thlen < sizeof(*th))
2176 		goto out;
2177 
2178 	if (!pskb_may_pull(skb, thlen))
2179 		goto out;
2180 
2181 	oldlen = (u16)~skb->len;
2182 	__skb_pull(skb, thlen);
2183 
2184 	if (skb_gso_ok(skb, features | NETIF_F_GSO_ROBUST)) {
2185 		/* Packet is from an untrusted source, reset gso_segs. */
2186 		int type = skb_shinfo(skb)->gso_type;
2187 		int mss;
2188 
2189 		if (unlikely(type &
2190 			     ~(SKB_GSO_TCPV4 |
2191 			       SKB_GSO_DODGY |
2192 			       SKB_GSO_TCP_ECN |
2193 			       SKB_GSO_TCPV6 |
2194 			       0) ||
2195 			     !(type & (SKB_GSO_TCPV4 | SKB_GSO_TCPV6))))
2196 			goto out;
2197 
2198 		mss = skb_shinfo(skb)->gso_size;
2199 		skb_shinfo(skb)->gso_segs = (skb->len + mss - 1) / mss;
2200 
2201 		segs = NULL;
2202 		goto out;
2203 	}
2204 
2205 	segs = skb_segment(skb, features);
2206 	if (IS_ERR(segs))
2207 		goto out;
2208 
2209 	len = skb_shinfo(skb)->gso_size;
2210 	delta = htonl(oldlen + (thlen + len));
2211 
2212 	skb = segs;
2213 	th = tcp_hdr(skb);
2214 	seq = ntohl(th->seq);
2215 
2216 	do {
2217 		th->fin = th->psh = 0;
2218 
2219 		th->check = ~csum_fold((__force __wsum)((__force u32)th->check +
2220 				       (__force u32)delta));
2221 		if (skb->ip_summed != CHECKSUM_PARTIAL)
2222 			th->check =
2223 			     csum_fold(csum_partial(skb_transport_header(skb),
2224 						    thlen, skb->csum));
2225 
2226 		seq += len;
2227 		skb = skb->next;
2228 		th = tcp_hdr(skb);
2229 
2230 		th->seq = htonl(seq);
2231 		th->cwr = 0;
2232 	} while (skb->next);
2233 
2234 	delta = htonl(oldlen + (skb->tail - skb->transport_header) +
2235 		      skb->data_len);
2236 	th->check = ~csum_fold((__force __wsum)((__force u32)th->check +
2237 				(__force u32)delta));
2238 	if (skb->ip_summed != CHECKSUM_PARTIAL)
2239 		th->check = csum_fold(csum_partial(skb_transport_header(skb),
2240 						   thlen, skb->csum));
2241 
2242 out:
2243 	return segs;
2244 }
2245 EXPORT_SYMBOL(tcp_tso_segment);
2246 
2247 #ifdef CONFIG_TCP_MD5SIG
2248 static unsigned long tcp_md5sig_users;
2249 static struct tcp_md5sig_pool **tcp_md5sig_pool;
2250 static DEFINE_SPINLOCK(tcp_md5sig_pool_lock);
2251 
2252 static void __tcp_free_md5sig_pool(struct tcp_md5sig_pool **pool)
2253 {
2254 	int cpu;
2255 	for_each_possible_cpu(cpu) {
2256 		struct tcp_md5sig_pool *p = *per_cpu_ptr(pool, cpu);
2257 		if (p) {
2258 			if (p->md5_desc.tfm)
2259 				crypto_free_hash(p->md5_desc.tfm);
2260 			kfree(p);
2261 			p = NULL;
2262 		}
2263 	}
2264 	free_percpu(pool);
2265 }
2266 
2267 void tcp_free_md5sig_pool(void)
2268 {
2269 	struct tcp_md5sig_pool **pool = NULL;
2270 
2271 	spin_lock_bh(&tcp_md5sig_pool_lock);
2272 	if (--tcp_md5sig_users == 0) {
2273 		pool = tcp_md5sig_pool;
2274 		tcp_md5sig_pool = NULL;
2275 	}
2276 	spin_unlock_bh(&tcp_md5sig_pool_lock);
2277 	if (pool)
2278 		__tcp_free_md5sig_pool(pool);
2279 }
2280 
2281 EXPORT_SYMBOL(tcp_free_md5sig_pool);
2282 
2283 static struct tcp_md5sig_pool **__tcp_alloc_md5sig_pool(void)
2284 {
2285 	int cpu;
2286 	struct tcp_md5sig_pool **pool;
2287 
2288 	pool = alloc_percpu(struct tcp_md5sig_pool *);
2289 	if (!pool)
2290 		return NULL;
2291 
2292 	for_each_possible_cpu(cpu) {
2293 		struct tcp_md5sig_pool *p;
2294 		struct crypto_hash *hash;
2295 
2296 		p = kzalloc(sizeof(*p), GFP_KERNEL);
2297 		if (!p)
2298 			goto out_free;
2299 		*per_cpu_ptr(pool, cpu) = p;
2300 
2301 		hash = crypto_alloc_hash("md5", 0, CRYPTO_ALG_ASYNC);
2302 		if (!hash || IS_ERR(hash))
2303 			goto out_free;
2304 
2305 		p->md5_desc.tfm = hash;
2306 	}
2307 	return pool;
2308 out_free:
2309 	__tcp_free_md5sig_pool(pool);
2310 	return NULL;
2311 }
2312 
2313 struct tcp_md5sig_pool **tcp_alloc_md5sig_pool(void)
2314 {
2315 	struct tcp_md5sig_pool **pool;
2316 	int alloc = 0;
2317 
2318 retry:
2319 	spin_lock_bh(&tcp_md5sig_pool_lock);
2320 	pool = tcp_md5sig_pool;
2321 	if (tcp_md5sig_users++ == 0) {
2322 		alloc = 1;
2323 		spin_unlock_bh(&tcp_md5sig_pool_lock);
2324 	} else if (!pool) {
2325 		tcp_md5sig_users--;
2326 		spin_unlock_bh(&tcp_md5sig_pool_lock);
2327 		cpu_relax();
2328 		goto retry;
2329 	} else
2330 		spin_unlock_bh(&tcp_md5sig_pool_lock);
2331 
2332 	if (alloc) {
2333 		/* we cannot hold spinlock here because this may sleep. */
2334 		struct tcp_md5sig_pool **p = __tcp_alloc_md5sig_pool();
2335 		spin_lock_bh(&tcp_md5sig_pool_lock);
2336 		if (!p) {
2337 			tcp_md5sig_users--;
2338 			spin_unlock_bh(&tcp_md5sig_pool_lock);
2339 			return NULL;
2340 		}
2341 		pool = tcp_md5sig_pool;
2342 		if (pool) {
2343 			/* oops, it has already been assigned. */
2344 			spin_unlock_bh(&tcp_md5sig_pool_lock);
2345 			__tcp_free_md5sig_pool(p);
2346 		} else {
2347 			tcp_md5sig_pool = pool = p;
2348 			spin_unlock_bh(&tcp_md5sig_pool_lock);
2349 		}
2350 	}
2351 	return pool;
2352 }
2353 
2354 EXPORT_SYMBOL(tcp_alloc_md5sig_pool);
2355 
2356 struct tcp_md5sig_pool *__tcp_get_md5sig_pool(int cpu)
2357 {
2358 	struct tcp_md5sig_pool **p;
2359 	spin_lock_bh(&tcp_md5sig_pool_lock);
2360 	p = tcp_md5sig_pool;
2361 	if (p)
2362 		tcp_md5sig_users++;
2363 	spin_unlock_bh(&tcp_md5sig_pool_lock);
2364 	return (p ? *per_cpu_ptr(p, cpu) : NULL);
2365 }
2366 
2367 EXPORT_SYMBOL(__tcp_get_md5sig_pool);
2368 
2369 void __tcp_put_md5sig_pool(void)
2370 {
2371 	tcp_free_md5sig_pool();
2372 }
2373 
2374 EXPORT_SYMBOL(__tcp_put_md5sig_pool);
2375 #endif
2376 
2377 void tcp_done(struct sock *sk)
2378 {
2379 	if(sk->sk_state == TCP_SYN_SENT || sk->sk_state == TCP_SYN_RECV)
2380 		TCP_INC_STATS_BH(TCP_MIB_ATTEMPTFAILS);
2381 
2382 	tcp_set_state(sk, TCP_CLOSE);
2383 	tcp_clear_xmit_timers(sk);
2384 
2385 	sk->sk_shutdown = SHUTDOWN_MASK;
2386 
2387 	if (!sock_flag(sk, SOCK_DEAD))
2388 		sk->sk_state_change(sk);
2389 	else
2390 		inet_csk_destroy_sock(sk);
2391 }
2392 EXPORT_SYMBOL_GPL(tcp_done);
2393 
2394 extern void __skb_cb_too_small_for_tcp(int, int);
2395 extern struct tcp_congestion_ops tcp_reno;
2396 
2397 static __initdata unsigned long thash_entries;
2398 static int __init set_thash_entries(char *str)
2399 {
2400 	if (!str)
2401 		return 0;
2402 	thash_entries = simple_strtoul(str, &str, 0);
2403 	return 1;
2404 }
2405 __setup("thash_entries=", set_thash_entries);
2406 
2407 void __init tcp_init(void)
2408 {
2409 	struct sk_buff *skb = NULL;
2410 	unsigned long limit;
2411 	int order, i, max_share;
2412 
2413 	if (sizeof(struct tcp_skb_cb) > sizeof(skb->cb))
2414 		__skb_cb_too_small_for_tcp(sizeof(struct tcp_skb_cb),
2415 					   sizeof(skb->cb));
2416 
2417 	tcp_hashinfo.bind_bucket_cachep =
2418 		kmem_cache_create("tcp_bind_bucket",
2419 				  sizeof(struct inet_bind_bucket), 0,
2420 				  SLAB_HWCACHE_ALIGN|SLAB_PANIC, NULL, NULL);
2421 
2422 	/* Size and allocate the main established and bind bucket
2423 	 * hash tables.
2424 	 *
2425 	 * The methodology is similar to that of the buffer cache.
2426 	 */
2427 	tcp_hashinfo.ehash =
2428 		alloc_large_system_hash("TCP established",
2429 					sizeof(struct inet_ehash_bucket),
2430 					thash_entries,
2431 					(num_physpages >= 128 * 1024) ?
2432 					13 : 15,
2433 					0,
2434 					&tcp_hashinfo.ehash_size,
2435 					NULL,
2436 					0);
2437 	tcp_hashinfo.ehash_size = 1 << tcp_hashinfo.ehash_size;
2438 	for (i = 0; i < tcp_hashinfo.ehash_size; i++) {
2439 		rwlock_init(&tcp_hashinfo.ehash[i].lock);
2440 		INIT_HLIST_HEAD(&tcp_hashinfo.ehash[i].chain);
2441 		INIT_HLIST_HEAD(&tcp_hashinfo.ehash[i].twchain);
2442 	}
2443 
2444 	tcp_hashinfo.bhash =
2445 		alloc_large_system_hash("TCP bind",
2446 					sizeof(struct inet_bind_hashbucket),
2447 					tcp_hashinfo.ehash_size,
2448 					(num_physpages >= 128 * 1024) ?
2449 					13 : 15,
2450 					0,
2451 					&tcp_hashinfo.bhash_size,
2452 					NULL,
2453 					64 * 1024);
2454 	tcp_hashinfo.bhash_size = 1 << tcp_hashinfo.bhash_size;
2455 	for (i = 0; i < tcp_hashinfo.bhash_size; i++) {
2456 		spin_lock_init(&tcp_hashinfo.bhash[i].lock);
2457 		INIT_HLIST_HEAD(&tcp_hashinfo.bhash[i].chain);
2458 	}
2459 
2460 	/* Try to be a bit smarter and adjust defaults depending
2461 	 * on available memory.
2462 	 */
2463 	for (order = 0; ((1 << order) << PAGE_SHIFT) <
2464 			(tcp_hashinfo.bhash_size * sizeof(struct inet_bind_hashbucket));
2465 			order++)
2466 		;
2467 	if (order >= 4) {
2468 		sysctl_local_port_range[0] = 32768;
2469 		sysctl_local_port_range[1] = 61000;
2470 		tcp_death_row.sysctl_max_tw_buckets = 180000;
2471 		sysctl_tcp_max_orphans = 4096 << (order - 4);
2472 		sysctl_max_syn_backlog = 1024;
2473 	} else if (order < 3) {
2474 		sysctl_local_port_range[0] = 1024 * (3 - order);
2475 		tcp_death_row.sysctl_max_tw_buckets >>= (3 - order);
2476 		sysctl_tcp_max_orphans >>= (3 - order);
2477 		sysctl_max_syn_backlog = 128;
2478 	}
2479 
2480 	/* Set the pressure threshold to be a fraction of global memory that
2481 	 * is up to 1/2 at 256 MB, decreasing toward zero with the amount of
2482 	 * memory, with a floor of 128 pages.
2483 	 */
2484 	limit = min(nr_all_pages, 1UL<<(28-PAGE_SHIFT)) >> (20-PAGE_SHIFT);
2485 	limit = (limit * (nr_all_pages >> (20-PAGE_SHIFT))) >> (PAGE_SHIFT-11);
2486 	limit = max(limit, 128UL);
2487 	sysctl_tcp_mem[0] = limit / 4 * 3;
2488 	sysctl_tcp_mem[1] = limit;
2489 	sysctl_tcp_mem[2] = sysctl_tcp_mem[0] * 2;
2490 
2491 	/* Set per-socket limits to no more than 1/128 the pressure threshold */
2492 	limit = ((unsigned long)sysctl_tcp_mem[1]) << (PAGE_SHIFT - 7);
2493 	max_share = min(4UL*1024*1024, limit);
2494 
2495 	sysctl_tcp_wmem[0] = SK_STREAM_MEM_QUANTUM;
2496 	sysctl_tcp_wmem[1] = 16*1024;
2497 	sysctl_tcp_wmem[2] = max(64*1024, max_share);
2498 
2499 	sysctl_tcp_rmem[0] = SK_STREAM_MEM_QUANTUM;
2500 	sysctl_tcp_rmem[1] = 87380;
2501 	sysctl_tcp_rmem[2] = max(87380, max_share);
2502 
2503 	printk(KERN_INFO "TCP: Hash tables configured "
2504 	       "(established %d bind %d)\n",
2505 	       tcp_hashinfo.ehash_size, tcp_hashinfo.bhash_size);
2506 
2507 	tcp_register_congestion_control(&tcp_reno);
2508 }
2509 
2510 EXPORT_SYMBOL(tcp_close);
2511 EXPORT_SYMBOL(tcp_disconnect);
2512 EXPORT_SYMBOL(tcp_getsockopt);
2513 EXPORT_SYMBOL(tcp_ioctl);
2514 EXPORT_SYMBOL(tcp_poll);
2515 EXPORT_SYMBOL(tcp_read_sock);
2516 EXPORT_SYMBOL(tcp_recvmsg);
2517 EXPORT_SYMBOL(tcp_sendmsg);
2518 EXPORT_SYMBOL(tcp_sendpage);
2519 EXPORT_SYMBOL(tcp_setsockopt);
2520 EXPORT_SYMBOL(tcp_shutdown);
2521 EXPORT_SYMBOL(tcp_statistics);
2522