1 /* 2 * INET An implementation of the TCP/IP protocol suite for the LINUX 3 * operating system. INET is implemented using the BSD Socket 4 * interface as the means of communication with the user level. 5 * 6 * Implementation of the Transmission Control Protocol(TCP). 7 * 8 * Authors: Ross Biro 9 * Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG> 10 * Mark Evans, <evansmp@uhura.aston.ac.uk> 11 * Corey Minyard <wf-rch!minyard@relay.EU.net> 12 * Florian La Roche, <flla@stud.uni-sb.de> 13 * Charles Hedrick, <hedrick@klinzhai.rutgers.edu> 14 * Linus Torvalds, <torvalds@cs.helsinki.fi> 15 * Alan Cox, <gw4pts@gw4pts.ampr.org> 16 * Matthew Dillon, <dillon@apollo.west.oic.com> 17 * Arnt Gulbrandsen, <agulbra@nvg.unit.no> 18 * Jorge Cwik, <jorge@laser.satlink.net> 19 * 20 * Fixes: 21 * Alan Cox : Numerous verify_area() calls 22 * Alan Cox : Set the ACK bit on a reset 23 * Alan Cox : Stopped it crashing if it closed while 24 * sk->inuse=1 and was trying to connect 25 * (tcp_err()). 26 * Alan Cox : All icmp error handling was broken 27 * pointers passed where wrong and the 28 * socket was looked up backwards. Nobody 29 * tested any icmp error code obviously. 30 * Alan Cox : tcp_err() now handled properly. It 31 * wakes people on errors. poll 32 * behaves and the icmp error race 33 * has gone by moving it into sock.c 34 * Alan Cox : tcp_send_reset() fixed to work for 35 * everything not just packets for 36 * unknown sockets. 37 * Alan Cox : tcp option processing. 38 * Alan Cox : Reset tweaked (still not 100%) [Had 39 * syn rule wrong] 40 * Herp Rosmanith : More reset fixes 41 * Alan Cox : No longer acks invalid rst frames. 42 * Acking any kind of RST is right out. 43 * Alan Cox : Sets an ignore me flag on an rst 44 * receive otherwise odd bits of prattle 45 * escape still 46 * Alan Cox : Fixed another acking RST frame bug. 47 * Should stop LAN workplace lockups. 48 * Alan Cox : Some tidyups using the new skb list 49 * facilities 50 * Alan Cox : sk->keepopen now seems to work 51 * Alan Cox : Pulls options out correctly on accepts 52 * Alan Cox : Fixed assorted sk->rqueue->next errors 53 * Alan Cox : PSH doesn't end a TCP read. Switched a 54 * bit to skb ops. 55 * Alan Cox : Tidied tcp_data to avoid a potential 56 * nasty. 57 * Alan Cox : Added some better commenting, as the 58 * tcp is hard to follow 59 * Alan Cox : Removed incorrect check for 20 * psh 60 * Michael O'Reilly : ack < copied bug fix. 61 * Johannes Stille : Misc tcp fixes (not all in yet). 62 * Alan Cox : FIN with no memory -> CRASH 63 * Alan Cox : Added socket option proto entries. 64 * Also added awareness of them to accept. 65 * Alan Cox : Added TCP options (SOL_TCP) 66 * Alan Cox : Switched wakeup calls to callbacks, 67 * so the kernel can layer network 68 * sockets. 69 * Alan Cox : Use ip_tos/ip_ttl settings. 70 * Alan Cox : Handle FIN (more) properly (we hope). 71 * Alan Cox : RST frames sent on unsynchronised 72 * state ack error. 73 * Alan Cox : Put in missing check for SYN bit. 74 * Alan Cox : Added tcp_select_window() aka NET2E 75 * window non shrink trick. 76 * Alan Cox : Added a couple of small NET2E timer 77 * fixes 78 * Charles Hedrick : TCP fixes 79 * Toomas Tamm : TCP window fixes 80 * Alan Cox : Small URG fix to rlogin ^C ack fight 81 * Charles Hedrick : Rewrote most of it to actually work 82 * Linus : Rewrote tcp_read() and URG handling 83 * completely 84 * Gerhard Koerting: Fixed some missing timer handling 85 * Matthew Dillon : Reworked TCP machine states as per RFC 86 * Gerhard Koerting: PC/TCP workarounds 87 * Adam Caldwell : Assorted timer/timing errors 88 * Matthew Dillon : Fixed another RST bug 89 * Alan Cox : Move to kernel side addressing changes. 90 * Alan Cox : Beginning work on TCP fastpathing 91 * (not yet usable) 92 * Arnt Gulbrandsen: Turbocharged tcp_check() routine. 93 * Alan Cox : TCP fast path debugging 94 * Alan Cox : Window clamping 95 * Michael Riepe : Bug in tcp_check() 96 * Matt Dillon : More TCP improvements and RST bug fixes 97 * Matt Dillon : Yet more small nasties remove from the 98 * TCP code (Be very nice to this man if 99 * tcp finally works 100%) 8) 100 * Alan Cox : BSD accept semantics. 101 * Alan Cox : Reset on closedown bug. 102 * Peter De Schrijver : ENOTCONN check missing in tcp_sendto(). 103 * Michael Pall : Handle poll() after URG properly in 104 * all cases. 105 * Michael Pall : Undo the last fix in tcp_read_urg() 106 * (multi URG PUSH broke rlogin). 107 * Michael Pall : Fix the multi URG PUSH problem in 108 * tcp_readable(), poll() after URG 109 * works now. 110 * Michael Pall : recv(...,MSG_OOB) never blocks in the 111 * BSD api. 112 * Alan Cox : Changed the semantics of sk->socket to 113 * fix a race and a signal problem with 114 * accept() and async I/O. 115 * Alan Cox : Relaxed the rules on tcp_sendto(). 116 * Yury Shevchuk : Really fixed accept() blocking problem. 117 * Craig I. Hagan : Allow for BSD compatible TIME_WAIT for 118 * clients/servers which listen in on 119 * fixed ports. 120 * Alan Cox : Cleaned the above up and shrank it to 121 * a sensible code size. 122 * Alan Cox : Self connect lockup fix. 123 * Alan Cox : No connect to multicast. 124 * Ross Biro : Close unaccepted children on master 125 * socket close. 126 * Alan Cox : Reset tracing code. 127 * Alan Cox : Spurious resets on shutdown. 128 * Alan Cox : Giant 15 minute/60 second timer error 129 * Alan Cox : Small whoops in polling before an 130 * accept. 131 * Alan Cox : Kept the state trace facility since 132 * it's handy for debugging. 133 * Alan Cox : More reset handler fixes. 134 * Alan Cox : Started rewriting the code based on 135 * the RFC's for other useful protocol 136 * references see: Comer, KA9Q NOS, and 137 * for a reference on the difference 138 * between specifications and how BSD 139 * works see the 4.4lite source. 140 * A.N.Kuznetsov : Don't time wait on completion of tidy 141 * close. 142 * Linus Torvalds : Fin/Shutdown & copied_seq changes. 143 * Linus Torvalds : Fixed BSD port reuse to work first syn 144 * Alan Cox : Reimplemented timers as per the RFC 145 * and using multiple timers for sanity. 146 * Alan Cox : Small bug fixes, and a lot of new 147 * comments. 148 * Alan Cox : Fixed dual reader crash by locking 149 * the buffers (much like datagram.c) 150 * Alan Cox : Fixed stuck sockets in probe. A probe 151 * now gets fed up of retrying without 152 * (even a no space) answer. 153 * Alan Cox : Extracted closing code better 154 * Alan Cox : Fixed the closing state machine to 155 * resemble the RFC. 156 * Alan Cox : More 'per spec' fixes. 157 * Jorge Cwik : Even faster checksumming. 158 * Alan Cox : tcp_data() doesn't ack illegal PSH 159 * only frames. At least one pc tcp stack 160 * generates them. 161 * Alan Cox : Cache last socket. 162 * Alan Cox : Per route irtt. 163 * Matt Day : poll()->select() match BSD precisely on error 164 * Alan Cox : New buffers 165 * Marc Tamsky : Various sk->prot->retransmits and 166 * sk->retransmits misupdating fixed. 167 * Fixed tcp_write_timeout: stuck close, 168 * and TCP syn retries gets used now. 169 * Mark Yarvis : In tcp_read_wakeup(), don't send an 170 * ack if state is TCP_CLOSED. 171 * Alan Cox : Look up device on a retransmit - routes may 172 * change. Doesn't yet cope with MSS shrink right 173 * but it's a start! 174 * Marc Tamsky : Closing in closing fixes. 175 * Mike Shaver : RFC1122 verifications. 176 * Alan Cox : rcv_saddr errors. 177 * Alan Cox : Block double connect(). 178 * Alan Cox : Small hooks for enSKIP. 179 * Alexey Kuznetsov: Path MTU discovery. 180 * Alan Cox : Support soft errors. 181 * Alan Cox : Fix MTU discovery pathological case 182 * when the remote claims no mtu! 183 * Marc Tamsky : TCP_CLOSE fix. 184 * Colin (G3TNE) : Send a reset on syn ack replies in 185 * window but wrong (fixes NT lpd problems) 186 * Pedro Roque : Better TCP window handling, delayed ack. 187 * Joerg Reuter : No modification of locked buffers in 188 * tcp_do_retransmit() 189 * Eric Schenk : Changed receiver side silly window 190 * avoidance algorithm to BSD style 191 * algorithm. This doubles throughput 192 * against machines running Solaris, 193 * and seems to result in general 194 * improvement. 195 * Stefan Magdalinski : adjusted tcp_readable() to fix FIONREAD 196 * Willy Konynenberg : Transparent proxying support. 197 * Mike McLagan : Routing by source 198 * Keith Owens : Do proper merging with partial SKB's in 199 * tcp_do_sendmsg to avoid burstiness. 200 * Eric Schenk : Fix fast close down bug with 201 * shutdown() followed by close(). 202 * Andi Kleen : Make poll agree with SIGIO 203 * Salvatore Sanfilippo : Support SO_LINGER with linger == 1 and 204 * lingertime == 0 (RFC 793 ABORT Call) 205 * Hirokazu Takahashi : Use copy_from_user() instead of 206 * csum_and_copy_from_user() if possible. 207 * 208 * This program is free software; you can redistribute it and/or 209 * modify it under the terms of the GNU General Public License 210 * as published by the Free Software Foundation; either version 211 * 2 of the License, or(at your option) any later version. 212 * 213 * Description of States: 214 * 215 * TCP_SYN_SENT sent a connection request, waiting for ack 216 * 217 * TCP_SYN_RECV received a connection request, sent ack, 218 * waiting for final ack in three-way handshake. 219 * 220 * TCP_ESTABLISHED connection established 221 * 222 * TCP_FIN_WAIT1 our side has shutdown, waiting to complete 223 * transmission of remaining buffered data 224 * 225 * TCP_FIN_WAIT2 all buffered data sent, waiting for remote 226 * to shutdown 227 * 228 * TCP_CLOSING both sides have shutdown but we still have 229 * data we have to finish sending 230 * 231 * TCP_TIME_WAIT timeout to catch resent junk before entering 232 * closed, can only be entered from FIN_WAIT2 233 * or CLOSING. Required because the other end 234 * may not have gotten our last ACK causing it 235 * to retransmit the data packet (which we ignore) 236 * 237 * TCP_CLOSE_WAIT remote side has shutdown and is waiting for 238 * us to finish writing our data and to shutdown 239 * (we have to close() to move on to LAST_ACK) 240 * 241 * TCP_LAST_ACK out side has shutdown after remote has 242 * shutdown. There may still be data in our 243 * buffer that we have to finish sending 244 * 245 * TCP_CLOSE socket is finished 246 */ 247 248 #define pr_fmt(fmt) "TCP: " fmt 249 250 #include <linux/kernel.h> 251 #include <linux/module.h> 252 #include <linux/types.h> 253 #include <linux/fcntl.h> 254 #include <linux/poll.h> 255 #include <linux/init.h> 256 #include <linux/fs.h> 257 #include <linux/skbuff.h> 258 #include <linux/scatterlist.h> 259 #include <linux/splice.h> 260 #include <linux/net.h> 261 #include <linux/socket.h> 262 #include <linux/random.h> 263 #include <linux/bootmem.h> 264 #include <linux/highmem.h> 265 #include <linux/swap.h> 266 #include <linux/cache.h> 267 #include <linux/err.h> 268 #include <linux/crypto.h> 269 #include <linux/time.h> 270 #include <linux/slab.h> 271 272 #include <net/icmp.h> 273 #include <net/inet_common.h> 274 #include <net/tcp.h> 275 #include <net/xfrm.h> 276 #include <net/ip.h> 277 #include <net/netdma.h> 278 #include <net/sock.h> 279 280 #include <asm/uaccess.h> 281 #include <asm/ioctls.h> 282 #include <net/busy_poll.h> 283 284 int sysctl_tcp_fin_timeout __read_mostly = TCP_FIN_TIMEOUT; 285 286 struct percpu_counter tcp_orphan_count; 287 EXPORT_SYMBOL_GPL(tcp_orphan_count); 288 289 int sysctl_tcp_wmem[3] __read_mostly; 290 int sysctl_tcp_rmem[3] __read_mostly; 291 292 EXPORT_SYMBOL(sysctl_tcp_rmem); 293 EXPORT_SYMBOL(sysctl_tcp_wmem); 294 295 atomic_long_t tcp_memory_allocated; /* Current allocated memory. */ 296 EXPORT_SYMBOL(tcp_memory_allocated); 297 298 /* 299 * Current number of TCP sockets. 300 */ 301 struct percpu_counter tcp_sockets_allocated; 302 EXPORT_SYMBOL(tcp_sockets_allocated); 303 304 /* 305 * TCP splice context 306 */ 307 struct tcp_splice_state { 308 struct pipe_inode_info *pipe; 309 size_t len; 310 unsigned int flags; 311 }; 312 313 /* 314 * Pressure flag: try to collapse. 315 * Technical note: it is used by multiple contexts non atomically. 316 * All the __sk_mem_schedule() is of this nature: accounting 317 * is strict, actions are advisory and have some latency. 318 */ 319 int tcp_memory_pressure __read_mostly; 320 EXPORT_SYMBOL(tcp_memory_pressure); 321 322 void tcp_enter_memory_pressure(struct sock *sk) 323 { 324 if (!tcp_memory_pressure) { 325 NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPMEMORYPRESSURES); 326 tcp_memory_pressure = 1; 327 } 328 } 329 EXPORT_SYMBOL(tcp_enter_memory_pressure); 330 331 /* Convert seconds to retransmits based on initial and max timeout */ 332 static u8 secs_to_retrans(int seconds, int timeout, int rto_max) 333 { 334 u8 res = 0; 335 336 if (seconds > 0) { 337 int period = timeout; 338 339 res = 1; 340 while (seconds > period && res < 255) { 341 res++; 342 timeout <<= 1; 343 if (timeout > rto_max) 344 timeout = rto_max; 345 period += timeout; 346 } 347 } 348 return res; 349 } 350 351 /* Convert retransmits to seconds based on initial and max timeout */ 352 static int retrans_to_secs(u8 retrans, int timeout, int rto_max) 353 { 354 int period = 0; 355 356 if (retrans > 0) { 357 period = timeout; 358 while (--retrans) { 359 timeout <<= 1; 360 if (timeout > rto_max) 361 timeout = rto_max; 362 period += timeout; 363 } 364 } 365 return period; 366 } 367 368 /* Address-family independent initialization for a tcp_sock. 369 * 370 * NOTE: A lot of things set to zero explicitly by call to 371 * sk_alloc() so need not be done here. 372 */ 373 void tcp_init_sock(struct sock *sk) 374 { 375 struct inet_connection_sock *icsk = inet_csk(sk); 376 struct tcp_sock *tp = tcp_sk(sk); 377 378 skb_queue_head_init(&tp->out_of_order_queue); 379 tcp_init_xmit_timers(sk); 380 tcp_prequeue_init(tp); 381 INIT_LIST_HEAD(&tp->tsq_node); 382 383 icsk->icsk_rto = TCP_TIMEOUT_INIT; 384 tp->mdev = TCP_TIMEOUT_INIT; 385 386 /* So many TCP implementations out there (incorrectly) count the 387 * initial SYN frame in their delayed-ACK and congestion control 388 * algorithms that we must have the following bandaid to talk 389 * efficiently to them. -DaveM 390 */ 391 tp->snd_cwnd = TCP_INIT_CWND; 392 393 /* See draft-stevens-tcpca-spec-01 for discussion of the 394 * initialization of these values. 395 */ 396 tp->snd_ssthresh = TCP_INFINITE_SSTHRESH; 397 tp->snd_cwnd_clamp = ~0; 398 tp->mss_cache = TCP_MSS_DEFAULT; 399 400 tp->reordering = sysctl_tcp_reordering; 401 tcp_enable_early_retrans(tp); 402 icsk->icsk_ca_ops = &tcp_init_congestion_ops; 403 404 tp->tsoffset = 0; 405 406 sk->sk_state = TCP_CLOSE; 407 408 sk->sk_write_space = sk_stream_write_space; 409 sock_set_flag(sk, SOCK_USE_WRITE_QUEUE); 410 411 icsk->icsk_sync_mss = tcp_sync_mss; 412 413 /* Presumed zeroed, in order of appearance: 414 * cookie_in_always, cookie_out_never, 415 * s_data_constant, s_data_in, s_data_out 416 */ 417 sk->sk_sndbuf = sysctl_tcp_wmem[1]; 418 sk->sk_rcvbuf = sysctl_tcp_rmem[1]; 419 420 local_bh_disable(); 421 sock_update_memcg(sk); 422 sk_sockets_allocated_inc(sk); 423 local_bh_enable(); 424 } 425 EXPORT_SYMBOL(tcp_init_sock); 426 427 /* 428 * Wait for a TCP event. 429 * 430 * Note that we don't need to lock the socket, as the upper poll layers 431 * take care of normal races (between the test and the event) and we don't 432 * go look at any of the socket buffers directly. 433 */ 434 unsigned int tcp_poll(struct file *file, struct socket *sock, poll_table *wait) 435 { 436 unsigned int mask; 437 struct sock *sk = sock->sk; 438 const struct tcp_sock *tp = tcp_sk(sk); 439 440 sock_rps_record_flow(sk); 441 442 sock_poll_wait(file, sk_sleep(sk), wait); 443 if (sk->sk_state == TCP_LISTEN) 444 return inet_csk_listen_poll(sk); 445 446 /* Socket is not locked. We are protected from async events 447 * by poll logic and correct handling of state changes 448 * made by other threads is impossible in any case. 449 */ 450 451 mask = 0; 452 453 /* 454 * POLLHUP is certainly not done right. But poll() doesn't 455 * have a notion of HUP in just one direction, and for a 456 * socket the read side is more interesting. 457 * 458 * Some poll() documentation says that POLLHUP is incompatible 459 * with the POLLOUT/POLLWR flags, so somebody should check this 460 * all. But careful, it tends to be safer to return too many 461 * bits than too few, and you can easily break real applications 462 * if you don't tell them that something has hung up! 463 * 464 * Check-me. 465 * 466 * Check number 1. POLLHUP is _UNMASKABLE_ event (see UNIX98 and 467 * our fs/select.c). It means that after we received EOF, 468 * poll always returns immediately, making impossible poll() on write() 469 * in state CLOSE_WAIT. One solution is evident --- to set POLLHUP 470 * if and only if shutdown has been made in both directions. 471 * Actually, it is interesting to look how Solaris and DUX 472 * solve this dilemma. I would prefer, if POLLHUP were maskable, 473 * then we could set it on SND_SHUTDOWN. BTW examples given 474 * in Stevens' books assume exactly this behaviour, it explains 475 * why POLLHUP is incompatible with POLLOUT. --ANK 476 * 477 * NOTE. Check for TCP_CLOSE is added. The goal is to prevent 478 * blocking on fresh not-connected or disconnected socket. --ANK 479 */ 480 if (sk->sk_shutdown == SHUTDOWN_MASK || sk->sk_state == TCP_CLOSE) 481 mask |= POLLHUP; 482 if (sk->sk_shutdown & RCV_SHUTDOWN) 483 mask |= POLLIN | POLLRDNORM | POLLRDHUP; 484 485 /* Connected or passive Fast Open socket? */ 486 if (sk->sk_state != TCP_SYN_SENT && 487 (sk->sk_state != TCP_SYN_RECV || tp->fastopen_rsk != NULL)) { 488 int target = sock_rcvlowat(sk, 0, INT_MAX); 489 490 if (tp->urg_seq == tp->copied_seq && 491 !sock_flag(sk, SOCK_URGINLINE) && 492 tp->urg_data) 493 target++; 494 495 /* Potential race condition. If read of tp below will 496 * escape above sk->sk_state, we can be illegally awaken 497 * in SYN_* states. */ 498 if (tp->rcv_nxt - tp->copied_seq >= target) 499 mask |= POLLIN | POLLRDNORM; 500 501 if (!(sk->sk_shutdown & SEND_SHUTDOWN)) { 502 if (sk_stream_wspace(sk) >= sk_stream_min_wspace(sk)) { 503 mask |= POLLOUT | POLLWRNORM; 504 } else { /* send SIGIO later */ 505 set_bit(SOCK_ASYNC_NOSPACE, 506 &sk->sk_socket->flags); 507 set_bit(SOCK_NOSPACE, &sk->sk_socket->flags); 508 509 /* Race breaker. If space is freed after 510 * wspace test but before the flags are set, 511 * IO signal will be lost. 512 */ 513 if (sk_stream_wspace(sk) >= sk_stream_min_wspace(sk)) 514 mask |= POLLOUT | POLLWRNORM; 515 } 516 } else 517 mask |= POLLOUT | POLLWRNORM; 518 519 if (tp->urg_data & TCP_URG_VALID) 520 mask |= POLLPRI; 521 } 522 /* This barrier is coupled with smp_wmb() in tcp_reset() */ 523 smp_rmb(); 524 if (sk->sk_err) 525 mask |= POLLERR; 526 527 return mask; 528 } 529 EXPORT_SYMBOL(tcp_poll); 530 531 int tcp_ioctl(struct sock *sk, int cmd, unsigned long arg) 532 { 533 struct tcp_sock *tp = tcp_sk(sk); 534 int answ; 535 bool slow; 536 537 switch (cmd) { 538 case SIOCINQ: 539 if (sk->sk_state == TCP_LISTEN) 540 return -EINVAL; 541 542 slow = lock_sock_fast(sk); 543 if ((1 << sk->sk_state) & (TCPF_SYN_SENT | TCPF_SYN_RECV)) 544 answ = 0; 545 else if (sock_flag(sk, SOCK_URGINLINE) || 546 !tp->urg_data || 547 before(tp->urg_seq, tp->copied_seq) || 548 !before(tp->urg_seq, tp->rcv_nxt)) { 549 550 answ = tp->rcv_nxt - tp->copied_seq; 551 552 /* Subtract 1, if FIN was received */ 553 if (answ && sock_flag(sk, SOCK_DONE)) 554 answ--; 555 } else 556 answ = tp->urg_seq - tp->copied_seq; 557 unlock_sock_fast(sk, slow); 558 break; 559 case SIOCATMARK: 560 answ = tp->urg_data && tp->urg_seq == tp->copied_seq; 561 break; 562 case SIOCOUTQ: 563 if (sk->sk_state == TCP_LISTEN) 564 return -EINVAL; 565 566 if ((1 << sk->sk_state) & (TCPF_SYN_SENT | TCPF_SYN_RECV)) 567 answ = 0; 568 else 569 answ = tp->write_seq - tp->snd_una; 570 break; 571 case SIOCOUTQNSD: 572 if (sk->sk_state == TCP_LISTEN) 573 return -EINVAL; 574 575 if ((1 << sk->sk_state) & (TCPF_SYN_SENT | TCPF_SYN_RECV)) 576 answ = 0; 577 else 578 answ = tp->write_seq - tp->snd_nxt; 579 break; 580 default: 581 return -ENOIOCTLCMD; 582 } 583 584 return put_user(answ, (int __user *)arg); 585 } 586 EXPORT_SYMBOL(tcp_ioctl); 587 588 static inline void tcp_mark_push(struct tcp_sock *tp, struct sk_buff *skb) 589 { 590 TCP_SKB_CB(skb)->tcp_flags |= TCPHDR_PSH; 591 tp->pushed_seq = tp->write_seq; 592 } 593 594 static inline bool forced_push(const struct tcp_sock *tp) 595 { 596 return after(tp->write_seq, tp->pushed_seq + (tp->max_window >> 1)); 597 } 598 599 static inline void skb_entail(struct sock *sk, struct sk_buff *skb) 600 { 601 struct tcp_sock *tp = tcp_sk(sk); 602 struct tcp_skb_cb *tcb = TCP_SKB_CB(skb); 603 604 skb->csum = 0; 605 tcb->seq = tcb->end_seq = tp->write_seq; 606 tcb->tcp_flags = TCPHDR_ACK; 607 tcb->sacked = 0; 608 skb_header_release(skb); 609 tcp_add_write_queue_tail(sk, skb); 610 sk->sk_wmem_queued += skb->truesize; 611 sk_mem_charge(sk, skb->truesize); 612 if (tp->nonagle & TCP_NAGLE_PUSH) 613 tp->nonagle &= ~TCP_NAGLE_PUSH; 614 } 615 616 static inline void tcp_mark_urg(struct tcp_sock *tp, int flags) 617 { 618 if (flags & MSG_OOB) 619 tp->snd_up = tp->write_seq; 620 } 621 622 static inline void tcp_push(struct sock *sk, int flags, int mss_now, 623 int nonagle) 624 { 625 if (tcp_send_head(sk)) { 626 struct tcp_sock *tp = tcp_sk(sk); 627 628 if (!(flags & MSG_MORE) || forced_push(tp)) 629 tcp_mark_push(tp, tcp_write_queue_tail(sk)); 630 631 tcp_mark_urg(tp, flags); 632 __tcp_push_pending_frames(sk, mss_now, 633 (flags & MSG_MORE) ? TCP_NAGLE_CORK : nonagle); 634 } 635 } 636 637 static int tcp_splice_data_recv(read_descriptor_t *rd_desc, struct sk_buff *skb, 638 unsigned int offset, size_t len) 639 { 640 struct tcp_splice_state *tss = rd_desc->arg.data; 641 int ret; 642 643 ret = skb_splice_bits(skb, offset, tss->pipe, min(rd_desc->count, len), 644 tss->flags); 645 if (ret > 0) 646 rd_desc->count -= ret; 647 return ret; 648 } 649 650 static int __tcp_splice_read(struct sock *sk, struct tcp_splice_state *tss) 651 { 652 /* Store TCP splice context information in read_descriptor_t. */ 653 read_descriptor_t rd_desc = { 654 .arg.data = tss, 655 .count = tss->len, 656 }; 657 658 return tcp_read_sock(sk, &rd_desc, tcp_splice_data_recv); 659 } 660 661 /** 662 * tcp_splice_read - splice data from TCP socket to a pipe 663 * @sock: socket to splice from 664 * @ppos: position (not valid) 665 * @pipe: pipe to splice to 666 * @len: number of bytes to splice 667 * @flags: splice modifier flags 668 * 669 * Description: 670 * Will read pages from given socket and fill them into a pipe. 671 * 672 **/ 673 ssize_t tcp_splice_read(struct socket *sock, loff_t *ppos, 674 struct pipe_inode_info *pipe, size_t len, 675 unsigned int flags) 676 { 677 struct sock *sk = sock->sk; 678 struct tcp_splice_state tss = { 679 .pipe = pipe, 680 .len = len, 681 .flags = flags, 682 }; 683 long timeo; 684 ssize_t spliced; 685 int ret; 686 687 sock_rps_record_flow(sk); 688 /* 689 * We can't seek on a socket input 690 */ 691 if (unlikely(*ppos)) 692 return -ESPIPE; 693 694 ret = spliced = 0; 695 696 lock_sock(sk); 697 698 timeo = sock_rcvtimeo(sk, sock->file->f_flags & O_NONBLOCK); 699 while (tss.len) { 700 ret = __tcp_splice_read(sk, &tss); 701 if (ret < 0) 702 break; 703 else if (!ret) { 704 if (spliced) 705 break; 706 if (sock_flag(sk, SOCK_DONE)) 707 break; 708 if (sk->sk_err) { 709 ret = sock_error(sk); 710 break; 711 } 712 if (sk->sk_shutdown & RCV_SHUTDOWN) 713 break; 714 if (sk->sk_state == TCP_CLOSE) { 715 /* 716 * This occurs when user tries to read 717 * from never connected socket. 718 */ 719 if (!sock_flag(sk, SOCK_DONE)) 720 ret = -ENOTCONN; 721 break; 722 } 723 if (!timeo) { 724 ret = -EAGAIN; 725 break; 726 } 727 sk_wait_data(sk, &timeo); 728 if (signal_pending(current)) { 729 ret = sock_intr_errno(timeo); 730 break; 731 } 732 continue; 733 } 734 tss.len -= ret; 735 spliced += ret; 736 737 if (!timeo) 738 break; 739 release_sock(sk); 740 lock_sock(sk); 741 742 if (sk->sk_err || sk->sk_state == TCP_CLOSE || 743 (sk->sk_shutdown & RCV_SHUTDOWN) || 744 signal_pending(current)) 745 break; 746 } 747 748 release_sock(sk); 749 750 if (spliced) 751 return spliced; 752 753 return ret; 754 } 755 EXPORT_SYMBOL(tcp_splice_read); 756 757 struct sk_buff *sk_stream_alloc_skb(struct sock *sk, int size, gfp_t gfp) 758 { 759 struct sk_buff *skb; 760 761 /* The TCP header must be at least 32-bit aligned. */ 762 size = ALIGN(size, 4); 763 764 skb = alloc_skb_fclone(size + sk->sk_prot->max_header, gfp); 765 if (skb) { 766 if (sk_wmem_schedule(sk, skb->truesize)) { 767 skb_reserve(skb, sk->sk_prot->max_header); 768 /* 769 * Make sure that we have exactly size bytes 770 * available to the caller, no more, no less. 771 */ 772 skb->reserved_tailroom = skb->end - skb->tail - size; 773 return skb; 774 } 775 __kfree_skb(skb); 776 } else { 777 sk->sk_prot->enter_memory_pressure(sk); 778 sk_stream_moderate_sndbuf(sk); 779 } 780 return NULL; 781 } 782 783 static unsigned int tcp_xmit_size_goal(struct sock *sk, u32 mss_now, 784 int large_allowed) 785 { 786 struct tcp_sock *tp = tcp_sk(sk); 787 u32 xmit_size_goal, old_size_goal; 788 789 xmit_size_goal = mss_now; 790 791 if (large_allowed && sk_can_gso(sk)) { 792 xmit_size_goal = ((sk->sk_gso_max_size - 1) - 793 inet_csk(sk)->icsk_af_ops->net_header_len - 794 inet_csk(sk)->icsk_ext_hdr_len - 795 tp->tcp_header_len); 796 797 /* TSQ : try to have two TSO segments in flight */ 798 xmit_size_goal = min_t(u32, xmit_size_goal, 799 sysctl_tcp_limit_output_bytes >> 1); 800 801 xmit_size_goal = tcp_bound_to_half_wnd(tp, xmit_size_goal); 802 803 /* We try hard to avoid divides here */ 804 old_size_goal = tp->xmit_size_goal_segs * mss_now; 805 806 if (likely(old_size_goal <= xmit_size_goal && 807 old_size_goal + mss_now > xmit_size_goal)) { 808 xmit_size_goal = old_size_goal; 809 } else { 810 tp->xmit_size_goal_segs = 811 min_t(u16, xmit_size_goal / mss_now, 812 sk->sk_gso_max_segs); 813 xmit_size_goal = tp->xmit_size_goal_segs * mss_now; 814 } 815 } 816 817 return max(xmit_size_goal, mss_now); 818 } 819 820 static int tcp_send_mss(struct sock *sk, int *size_goal, int flags) 821 { 822 int mss_now; 823 824 mss_now = tcp_current_mss(sk); 825 *size_goal = tcp_xmit_size_goal(sk, mss_now, !(flags & MSG_OOB)); 826 827 return mss_now; 828 } 829 830 static ssize_t do_tcp_sendpages(struct sock *sk, struct page *page, int offset, 831 size_t size, int flags) 832 { 833 struct tcp_sock *tp = tcp_sk(sk); 834 int mss_now, size_goal; 835 int err; 836 ssize_t copied; 837 long timeo = sock_sndtimeo(sk, flags & MSG_DONTWAIT); 838 839 /* Wait for a connection to finish. One exception is TCP Fast Open 840 * (passive side) where data is allowed to be sent before a connection 841 * is fully established. 842 */ 843 if (((1 << sk->sk_state) & ~(TCPF_ESTABLISHED | TCPF_CLOSE_WAIT)) && 844 !tcp_passive_fastopen(sk)) { 845 if ((err = sk_stream_wait_connect(sk, &timeo)) != 0) 846 goto out_err; 847 } 848 849 clear_bit(SOCK_ASYNC_NOSPACE, &sk->sk_socket->flags); 850 851 mss_now = tcp_send_mss(sk, &size_goal, flags); 852 copied = 0; 853 854 err = -EPIPE; 855 if (sk->sk_err || (sk->sk_shutdown & SEND_SHUTDOWN)) 856 goto out_err; 857 858 while (size > 0) { 859 struct sk_buff *skb = tcp_write_queue_tail(sk); 860 int copy, i; 861 bool can_coalesce; 862 863 if (!tcp_send_head(sk) || (copy = size_goal - skb->len) <= 0) { 864 new_segment: 865 if (!sk_stream_memory_free(sk)) 866 goto wait_for_sndbuf; 867 868 skb = sk_stream_alloc_skb(sk, 0, sk->sk_allocation); 869 if (!skb) 870 goto wait_for_memory; 871 872 skb_entail(sk, skb); 873 copy = size_goal; 874 } 875 876 if (copy > size) 877 copy = size; 878 879 i = skb_shinfo(skb)->nr_frags; 880 can_coalesce = skb_can_coalesce(skb, i, page, offset); 881 if (!can_coalesce && i >= MAX_SKB_FRAGS) { 882 tcp_mark_push(tp, skb); 883 goto new_segment; 884 } 885 if (!sk_wmem_schedule(sk, copy)) 886 goto wait_for_memory; 887 888 if (can_coalesce) { 889 skb_frag_size_add(&skb_shinfo(skb)->frags[i - 1], copy); 890 } else { 891 get_page(page); 892 skb_fill_page_desc(skb, i, page, offset, copy); 893 } 894 skb_shinfo(skb)->tx_flags |= SKBTX_SHARED_FRAG; 895 896 skb->len += copy; 897 skb->data_len += copy; 898 skb->truesize += copy; 899 sk->sk_wmem_queued += copy; 900 sk_mem_charge(sk, copy); 901 skb->ip_summed = CHECKSUM_PARTIAL; 902 tp->write_seq += copy; 903 TCP_SKB_CB(skb)->end_seq += copy; 904 skb_shinfo(skb)->gso_segs = 0; 905 906 if (!copied) 907 TCP_SKB_CB(skb)->tcp_flags &= ~TCPHDR_PSH; 908 909 copied += copy; 910 offset += copy; 911 if (!(size -= copy)) 912 goto out; 913 914 if (skb->len < size_goal || (flags & MSG_OOB)) 915 continue; 916 917 if (forced_push(tp)) { 918 tcp_mark_push(tp, skb); 919 __tcp_push_pending_frames(sk, mss_now, TCP_NAGLE_PUSH); 920 } else if (skb == tcp_send_head(sk)) 921 tcp_push_one(sk, mss_now); 922 continue; 923 924 wait_for_sndbuf: 925 set_bit(SOCK_NOSPACE, &sk->sk_socket->flags); 926 wait_for_memory: 927 tcp_push(sk, flags & ~MSG_MORE, mss_now, TCP_NAGLE_PUSH); 928 929 if ((err = sk_stream_wait_memory(sk, &timeo)) != 0) 930 goto do_error; 931 932 mss_now = tcp_send_mss(sk, &size_goal, flags); 933 } 934 935 out: 936 if (copied && !(flags & MSG_SENDPAGE_NOTLAST)) 937 tcp_push(sk, flags, mss_now, tp->nonagle); 938 return copied; 939 940 do_error: 941 if (copied) 942 goto out; 943 out_err: 944 return sk_stream_error(sk, flags, err); 945 } 946 947 int tcp_sendpage(struct sock *sk, struct page *page, int offset, 948 size_t size, int flags) 949 { 950 ssize_t res; 951 952 if (!(sk->sk_route_caps & NETIF_F_SG) || 953 !(sk->sk_route_caps & NETIF_F_ALL_CSUM)) 954 return sock_no_sendpage(sk->sk_socket, page, offset, size, 955 flags); 956 957 lock_sock(sk); 958 res = do_tcp_sendpages(sk, page, offset, size, flags); 959 release_sock(sk); 960 return res; 961 } 962 EXPORT_SYMBOL(tcp_sendpage); 963 964 static inline int select_size(const struct sock *sk, bool sg) 965 { 966 const struct tcp_sock *tp = tcp_sk(sk); 967 int tmp = tp->mss_cache; 968 969 if (sg) { 970 if (sk_can_gso(sk)) { 971 /* Small frames wont use a full page: 972 * Payload will immediately follow tcp header. 973 */ 974 tmp = SKB_WITH_OVERHEAD(2048 - MAX_TCP_HEADER); 975 } else { 976 int pgbreak = SKB_MAX_HEAD(MAX_TCP_HEADER); 977 978 if (tmp >= pgbreak && 979 tmp <= pgbreak + (MAX_SKB_FRAGS - 1) * PAGE_SIZE) 980 tmp = pgbreak; 981 } 982 } 983 984 return tmp; 985 } 986 987 void tcp_free_fastopen_req(struct tcp_sock *tp) 988 { 989 if (tp->fastopen_req != NULL) { 990 kfree(tp->fastopen_req); 991 tp->fastopen_req = NULL; 992 } 993 } 994 995 static int tcp_sendmsg_fastopen(struct sock *sk, struct msghdr *msg, int *size) 996 { 997 struct tcp_sock *tp = tcp_sk(sk); 998 int err, flags; 999 1000 if (!(sysctl_tcp_fastopen & TFO_CLIENT_ENABLE)) 1001 return -EOPNOTSUPP; 1002 if (tp->fastopen_req != NULL) 1003 return -EALREADY; /* Another Fast Open is in progress */ 1004 1005 tp->fastopen_req = kzalloc(sizeof(struct tcp_fastopen_request), 1006 sk->sk_allocation); 1007 if (unlikely(tp->fastopen_req == NULL)) 1008 return -ENOBUFS; 1009 tp->fastopen_req->data = msg; 1010 1011 flags = (msg->msg_flags & MSG_DONTWAIT) ? O_NONBLOCK : 0; 1012 err = __inet_stream_connect(sk->sk_socket, msg->msg_name, 1013 msg->msg_namelen, flags); 1014 *size = tp->fastopen_req->copied; 1015 tcp_free_fastopen_req(tp); 1016 return err; 1017 } 1018 1019 int tcp_sendmsg(struct kiocb *iocb, struct sock *sk, struct msghdr *msg, 1020 size_t size) 1021 { 1022 struct iovec *iov; 1023 struct tcp_sock *tp = tcp_sk(sk); 1024 struct sk_buff *skb; 1025 int iovlen, flags, err, copied = 0; 1026 int mss_now = 0, size_goal, copied_syn = 0, offset = 0; 1027 bool sg; 1028 long timeo; 1029 1030 lock_sock(sk); 1031 1032 flags = msg->msg_flags; 1033 if (flags & MSG_FASTOPEN) { 1034 err = tcp_sendmsg_fastopen(sk, msg, &copied_syn); 1035 if (err == -EINPROGRESS && copied_syn > 0) 1036 goto out; 1037 else if (err) 1038 goto out_err; 1039 offset = copied_syn; 1040 } 1041 1042 timeo = sock_sndtimeo(sk, flags & MSG_DONTWAIT); 1043 1044 /* Wait for a connection to finish. One exception is TCP Fast Open 1045 * (passive side) where data is allowed to be sent before a connection 1046 * is fully established. 1047 */ 1048 if (((1 << sk->sk_state) & ~(TCPF_ESTABLISHED | TCPF_CLOSE_WAIT)) && 1049 !tcp_passive_fastopen(sk)) { 1050 if ((err = sk_stream_wait_connect(sk, &timeo)) != 0) 1051 goto do_error; 1052 } 1053 1054 if (unlikely(tp->repair)) { 1055 if (tp->repair_queue == TCP_RECV_QUEUE) { 1056 copied = tcp_send_rcvq(sk, msg, size); 1057 goto out; 1058 } 1059 1060 err = -EINVAL; 1061 if (tp->repair_queue == TCP_NO_QUEUE) 1062 goto out_err; 1063 1064 /* 'common' sending to sendq */ 1065 } 1066 1067 /* This should be in poll */ 1068 clear_bit(SOCK_ASYNC_NOSPACE, &sk->sk_socket->flags); 1069 1070 mss_now = tcp_send_mss(sk, &size_goal, flags); 1071 1072 /* Ok commence sending. */ 1073 iovlen = msg->msg_iovlen; 1074 iov = msg->msg_iov; 1075 copied = 0; 1076 1077 err = -EPIPE; 1078 if (sk->sk_err || (sk->sk_shutdown & SEND_SHUTDOWN)) 1079 goto out_err; 1080 1081 sg = !!(sk->sk_route_caps & NETIF_F_SG); 1082 1083 while (--iovlen >= 0) { 1084 size_t seglen = iov->iov_len; 1085 unsigned char __user *from = iov->iov_base; 1086 1087 iov++; 1088 if (unlikely(offset > 0)) { /* Skip bytes copied in SYN */ 1089 if (offset >= seglen) { 1090 offset -= seglen; 1091 continue; 1092 } 1093 seglen -= offset; 1094 from += offset; 1095 offset = 0; 1096 } 1097 1098 while (seglen > 0) { 1099 int copy = 0; 1100 int max = size_goal; 1101 1102 skb = tcp_write_queue_tail(sk); 1103 if (tcp_send_head(sk)) { 1104 if (skb->ip_summed == CHECKSUM_NONE) 1105 max = mss_now; 1106 copy = max - skb->len; 1107 } 1108 1109 if (copy <= 0) { 1110 new_segment: 1111 /* Allocate new segment. If the interface is SG, 1112 * allocate skb fitting to single page. 1113 */ 1114 if (!sk_stream_memory_free(sk)) 1115 goto wait_for_sndbuf; 1116 1117 skb = sk_stream_alloc_skb(sk, 1118 select_size(sk, sg), 1119 sk->sk_allocation); 1120 if (!skb) 1121 goto wait_for_memory; 1122 1123 /* 1124 * Check whether we can use HW checksum. 1125 */ 1126 if (sk->sk_route_caps & NETIF_F_ALL_CSUM) 1127 skb->ip_summed = CHECKSUM_PARTIAL; 1128 1129 skb_entail(sk, skb); 1130 copy = size_goal; 1131 max = size_goal; 1132 } 1133 1134 /* Try to append data to the end of skb. */ 1135 if (copy > seglen) 1136 copy = seglen; 1137 1138 /* Where to copy to? */ 1139 if (skb_availroom(skb) > 0) { 1140 /* We have some space in skb head. Superb! */ 1141 copy = min_t(int, copy, skb_availroom(skb)); 1142 err = skb_add_data_nocache(sk, skb, from, copy); 1143 if (err) 1144 goto do_fault; 1145 } else { 1146 bool merge = true; 1147 int i = skb_shinfo(skb)->nr_frags; 1148 struct page_frag *pfrag = sk_page_frag(sk); 1149 1150 if (!sk_page_frag_refill(sk, pfrag)) 1151 goto wait_for_memory; 1152 1153 if (!skb_can_coalesce(skb, i, pfrag->page, 1154 pfrag->offset)) { 1155 if (i == MAX_SKB_FRAGS || !sg) { 1156 tcp_mark_push(tp, skb); 1157 goto new_segment; 1158 } 1159 merge = false; 1160 } 1161 1162 copy = min_t(int, copy, pfrag->size - pfrag->offset); 1163 1164 if (!sk_wmem_schedule(sk, copy)) 1165 goto wait_for_memory; 1166 1167 err = skb_copy_to_page_nocache(sk, from, skb, 1168 pfrag->page, 1169 pfrag->offset, 1170 copy); 1171 if (err) 1172 goto do_error; 1173 1174 /* Update the skb. */ 1175 if (merge) { 1176 skb_frag_size_add(&skb_shinfo(skb)->frags[i - 1], copy); 1177 } else { 1178 skb_fill_page_desc(skb, i, pfrag->page, 1179 pfrag->offset, copy); 1180 get_page(pfrag->page); 1181 } 1182 pfrag->offset += copy; 1183 } 1184 1185 if (!copied) 1186 TCP_SKB_CB(skb)->tcp_flags &= ~TCPHDR_PSH; 1187 1188 tp->write_seq += copy; 1189 TCP_SKB_CB(skb)->end_seq += copy; 1190 skb_shinfo(skb)->gso_segs = 0; 1191 1192 from += copy; 1193 copied += copy; 1194 if ((seglen -= copy) == 0 && iovlen == 0) 1195 goto out; 1196 1197 if (skb->len < max || (flags & MSG_OOB) || unlikely(tp->repair)) 1198 continue; 1199 1200 if (forced_push(tp)) { 1201 tcp_mark_push(tp, skb); 1202 __tcp_push_pending_frames(sk, mss_now, TCP_NAGLE_PUSH); 1203 } else if (skb == tcp_send_head(sk)) 1204 tcp_push_one(sk, mss_now); 1205 continue; 1206 1207 wait_for_sndbuf: 1208 set_bit(SOCK_NOSPACE, &sk->sk_socket->flags); 1209 wait_for_memory: 1210 if (copied) 1211 tcp_push(sk, flags & ~MSG_MORE, mss_now, TCP_NAGLE_PUSH); 1212 1213 if ((err = sk_stream_wait_memory(sk, &timeo)) != 0) 1214 goto do_error; 1215 1216 mss_now = tcp_send_mss(sk, &size_goal, flags); 1217 } 1218 } 1219 1220 out: 1221 if (copied) 1222 tcp_push(sk, flags, mss_now, tp->nonagle); 1223 release_sock(sk); 1224 return copied + copied_syn; 1225 1226 do_fault: 1227 if (!skb->len) { 1228 tcp_unlink_write_queue(skb, sk); 1229 /* It is the one place in all of TCP, except connection 1230 * reset, where we can be unlinking the send_head. 1231 */ 1232 tcp_check_send_head(sk, skb); 1233 sk_wmem_free_skb(sk, skb); 1234 } 1235 1236 do_error: 1237 if (copied + copied_syn) 1238 goto out; 1239 out_err: 1240 err = sk_stream_error(sk, flags, err); 1241 release_sock(sk); 1242 return err; 1243 } 1244 EXPORT_SYMBOL(tcp_sendmsg); 1245 1246 /* 1247 * Handle reading urgent data. BSD has very simple semantics for 1248 * this, no blocking and very strange errors 8) 1249 */ 1250 1251 static int tcp_recv_urg(struct sock *sk, struct msghdr *msg, int len, int flags) 1252 { 1253 struct tcp_sock *tp = tcp_sk(sk); 1254 1255 /* No URG data to read. */ 1256 if (sock_flag(sk, SOCK_URGINLINE) || !tp->urg_data || 1257 tp->urg_data == TCP_URG_READ) 1258 return -EINVAL; /* Yes this is right ! */ 1259 1260 if (sk->sk_state == TCP_CLOSE && !sock_flag(sk, SOCK_DONE)) 1261 return -ENOTCONN; 1262 1263 if (tp->urg_data & TCP_URG_VALID) { 1264 int err = 0; 1265 char c = tp->urg_data; 1266 1267 if (!(flags & MSG_PEEK)) 1268 tp->urg_data = TCP_URG_READ; 1269 1270 /* Read urgent data. */ 1271 msg->msg_flags |= MSG_OOB; 1272 1273 if (len > 0) { 1274 if (!(flags & MSG_TRUNC)) 1275 err = memcpy_toiovec(msg->msg_iov, &c, 1); 1276 len = 1; 1277 } else 1278 msg->msg_flags |= MSG_TRUNC; 1279 1280 return err ? -EFAULT : len; 1281 } 1282 1283 if (sk->sk_state == TCP_CLOSE || (sk->sk_shutdown & RCV_SHUTDOWN)) 1284 return 0; 1285 1286 /* Fixed the recv(..., MSG_OOB) behaviour. BSD docs and 1287 * the available implementations agree in this case: 1288 * this call should never block, independent of the 1289 * blocking state of the socket. 1290 * Mike <pall@rz.uni-karlsruhe.de> 1291 */ 1292 return -EAGAIN; 1293 } 1294 1295 static int tcp_peek_sndq(struct sock *sk, struct msghdr *msg, int len) 1296 { 1297 struct sk_buff *skb; 1298 int copied = 0, err = 0; 1299 1300 /* XXX -- need to support SO_PEEK_OFF */ 1301 1302 skb_queue_walk(&sk->sk_write_queue, skb) { 1303 err = skb_copy_datagram_iovec(skb, 0, msg->msg_iov, skb->len); 1304 if (err) 1305 break; 1306 1307 copied += skb->len; 1308 } 1309 1310 return err ?: copied; 1311 } 1312 1313 /* Clean up the receive buffer for full frames taken by the user, 1314 * then send an ACK if necessary. COPIED is the number of bytes 1315 * tcp_recvmsg has given to the user so far, it speeds up the 1316 * calculation of whether or not we must ACK for the sake of 1317 * a window update. 1318 */ 1319 void tcp_cleanup_rbuf(struct sock *sk, int copied) 1320 { 1321 struct tcp_sock *tp = tcp_sk(sk); 1322 bool time_to_ack = false; 1323 1324 struct sk_buff *skb = skb_peek(&sk->sk_receive_queue); 1325 1326 WARN(skb && !before(tp->copied_seq, TCP_SKB_CB(skb)->end_seq), 1327 "cleanup rbuf bug: copied %X seq %X rcvnxt %X\n", 1328 tp->copied_seq, TCP_SKB_CB(skb)->end_seq, tp->rcv_nxt); 1329 1330 if (inet_csk_ack_scheduled(sk)) { 1331 const struct inet_connection_sock *icsk = inet_csk(sk); 1332 /* Delayed ACKs frequently hit locked sockets during bulk 1333 * receive. */ 1334 if (icsk->icsk_ack.blocked || 1335 /* Once-per-two-segments ACK was not sent by tcp_input.c */ 1336 tp->rcv_nxt - tp->rcv_wup > icsk->icsk_ack.rcv_mss || 1337 /* 1338 * If this read emptied read buffer, we send ACK, if 1339 * connection is not bidirectional, user drained 1340 * receive buffer and there was a small segment 1341 * in queue. 1342 */ 1343 (copied > 0 && 1344 ((icsk->icsk_ack.pending & ICSK_ACK_PUSHED2) || 1345 ((icsk->icsk_ack.pending & ICSK_ACK_PUSHED) && 1346 !icsk->icsk_ack.pingpong)) && 1347 !atomic_read(&sk->sk_rmem_alloc))) 1348 time_to_ack = true; 1349 } 1350 1351 /* We send an ACK if we can now advertise a non-zero window 1352 * which has been raised "significantly". 1353 * 1354 * Even if window raised up to infinity, do not send window open ACK 1355 * in states, where we will not receive more. It is useless. 1356 */ 1357 if (copied > 0 && !time_to_ack && !(sk->sk_shutdown & RCV_SHUTDOWN)) { 1358 __u32 rcv_window_now = tcp_receive_window(tp); 1359 1360 /* Optimize, __tcp_select_window() is not cheap. */ 1361 if (2*rcv_window_now <= tp->window_clamp) { 1362 __u32 new_window = __tcp_select_window(sk); 1363 1364 /* Send ACK now, if this read freed lots of space 1365 * in our buffer. Certainly, new_window is new window. 1366 * We can advertise it now, if it is not less than current one. 1367 * "Lots" means "at least twice" here. 1368 */ 1369 if (new_window && new_window >= 2 * rcv_window_now) 1370 time_to_ack = true; 1371 } 1372 } 1373 if (time_to_ack) 1374 tcp_send_ack(sk); 1375 } 1376 1377 static void tcp_prequeue_process(struct sock *sk) 1378 { 1379 struct sk_buff *skb; 1380 struct tcp_sock *tp = tcp_sk(sk); 1381 1382 NET_INC_STATS_USER(sock_net(sk), LINUX_MIB_TCPPREQUEUED); 1383 1384 /* RX process wants to run with disabled BHs, though it is not 1385 * necessary */ 1386 local_bh_disable(); 1387 while ((skb = __skb_dequeue(&tp->ucopy.prequeue)) != NULL) 1388 sk_backlog_rcv(sk, skb); 1389 local_bh_enable(); 1390 1391 /* Clear memory counter. */ 1392 tp->ucopy.memory = 0; 1393 } 1394 1395 #ifdef CONFIG_NET_DMA 1396 static void tcp_service_net_dma(struct sock *sk, bool wait) 1397 { 1398 dma_cookie_t done, used; 1399 dma_cookie_t last_issued; 1400 struct tcp_sock *tp = tcp_sk(sk); 1401 1402 if (!tp->ucopy.dma_chan) 1403 return; 1404 1405 last_issued = tp->ucopy.dma_cookie; 1406 dma_async_issue_pending(tp->ucopy.dma_chan); 1407 1408 do { 1409 if (dma_async_is_tx_complete(tp->ucopy.dma_chan, 1410 last_issued, &done, 1411 &used) == DMA_SUCCESS) { 1412 /* Safe to free early-copied skbs now */ 1413 __skb_queue_purge(&sk->sk_async_wait_queue); 1414 break; 1415 } else { 1416 struct sk_buff *skb; 1417 while ((skb = skb_peek(&sk->sk_async_wait_queue)) && 1418 (dma_async_is_complete(skb->dma_cookie, done, 1419 used) == DMA_SUCCESS)) { 1420 __skb_dequeue(&sk->sk_async_wait_queue); 1421 kfree_skb(skb); 1422 } 1423 } 1424 } while (wait); 1425 } 1426 #endif 1427 1428 static struct sk_buff *tcp_recv_skb(struct sock *sk, u32 seq, u32 *off) 1429 { 1430 struct sk_buff *skb; 1431 u32 offset; 1432 1433 while ((skb = skb_peek(&sk->sk_receive_queue)) != NULL) { 1434 offset = seq - TCP_SKB_CB(skb)->seq; 1435 if (tcp_hdr(skb)->syn) 1436 offset--; 1437 if (offset < skb->len || tcp_hdr(skb)->fin) { 1438 *off = offset; 1439 return skb; 1440 } 1441 /* This looks weird, but this can happen if TCP collapsing 1442 * splitted a fat GRO packet, while we released socket lock 1443 * in skb_splice_bits() 1444 */ 1445 sk_eat_skb(sk, skb, false); 1446 } 1447 return NULL; 1448 } 1449 1450 /* 1451 * This routine provides an alternative to tcp_recvmsg() for routines 1452 * that would like to handle copying from skbuffs directly in 'sendfile' 1453 * fashion. 1454 * Note: 1455 * - It is assumed that the socket was locked by the caller. 1456 * - The routine does not block. 1457 * - At present, there is no support for reading OOB data 1458 * or for 'peeking' the socket using this routine 1459 * (although both would be easy to implement). 1460 */ 1461 int tcp_read_sock(struct sock *sk, read_descriptor_t *desc, 1462 sk_read_actor_t recv_actor) 1463 { 1464 struct sk_buff *skb; 1465 struct tcp_sock *tp = tcp_sk(sk); 1466 u32 seq = tp->copied_seq; 1467 u32 offset; 1468 int copied = 0; 1469 1470 if (sk->sk_state == TCP_LISTEN) 1471 return -ENOTCONN; 1472 while ((skb = tcp_recv_skb(sk, seq, &offset)) != NULL) { 1473 if (offset < skb->len) { 1474 int used; 1475 size_t len; 1476 1477 len = skb->len - offset; 1478 /* Stop reading if we hit a patch of urgent data */ 1479 if (tp->urg_data) { 1480 u32 urg_offset = tp->urg_seq - seq; 1481 if (urg_offset < len) 1482 len = urg_offset; 1483 if (!len) 1484 break; 1485 } 1486 used = recv_actor(desc, skb, offset, len); 1487 if (used <= 0) { 1488 if (!copied) 1489 copied = used; 1490 break; 1491 } else if (used <= len) { 1492 seq += used; 1493 copied += used; 1494 offset += used; 1495 } 1496 /* If recv_actor drops the lock (e.g. TCP splice 1497 * receive) the skb pointer might be invalid when 1498 * getting here: tcp_collapse might have deleted it 1499 * while aggregating skbs from the socket queue. 1500 */ 1501 skb = tcp_recv_skb(sk, seq - 1, &offset); 1502 if (!skb) 1503 break; 1504 /* TCP coalescing might have appended data to the skb. 1505 * Try to splice more frags 1506 */ 1507 if (offset + 1 != skb->len) 1508 continue; 1509 } 1510 if (tcp_hdr(skb)->fin) { 1511 sk_eat_skb(sk, skb, false); 1512 ++seq; 1513 break; 1514 } 1515 sk_eat_skb(sk, skb, false); 1516 if (!desc->count) 1517 break; 1518 tp->copied_seq = seq; 1519 } 1520 tp->copied_seq = seq; 1521 1522 tcp_rcv_space_adjust(sk); 1523 1524 /* Clean up data we have read: This will do ACK frames. */ 1525 if (copied > 0) { 1526 tcp_recv_skb(sk, seq, &offset); 1527 tcp_cleanup_rbuf(sk, copied); 1528 } 1529 return copied; 1530 } 1531 EXPORT_SYMBOL(tcp_read_sock); 1532 1533 /* 1534 * This routine copies from a sock struct into the user buffer. 1535 * 1536 * Technical note: in 2.3 we work on _locked_ socket, so that 1537 * tricks with *seq access order and skb->users are not required. 1538 * Probably, code can be easily improved even more. 1539 */ 1540 1541 int tcp_recvmsg(struct kiocb *iocb, struct sock *sk, struct msghdr *msg, 1542 size_t len, int nonblock, int flags, int *addr_len) 1543 { 1544 struct tcp_sock *tp = tcp_sk(sk); 1545 int copied = 0; 1546 u32 peek_seq; 1547 u32 *seq; 1548 unsigned long used; 1549 int err; 1550 int target; /* Read at least this many bytes */ 1551 long timeo; 1552 struct task_struct *user_recv = NULL; 1553 bool copied_early = false; 1554 struct sk_buff *skb; 1555 u32 urg_hole = 0; 1556 1557 if (sk_can_busy_loop(sk) && skb_queue_empty(&sk->sk_receive_queue) && 1558 (sk->sk_state == TCP_ESTABLISHED)) 1559 sk_busy_loop(sk, nonblock); 1560 1561 lock_sock(sk); 1562 1563 err = -ENOTCONN; 1564 if (sk->sk_state == TCP_LISTEN) 1565 goto out; 1566 1567 timeo = sock_rcvtimeo(sk, nonblock); 1568 1569 /* Urgent data needs to be handled specially. */ 1570 if (flags & MSG_OOB) 1571 goto recv_urg; 1572 1573 if (unlikely(tp->repair)) { 1574 err = -EPERM; 1575 if (!(flags & MSG_PEEK)) 1576 goto out; 1577 1578 if (tp->repair_queue == TCP_SEND_QUEUE) 1579 goto recv_sndq; 1580 1581 err = -EINVAL; 1582 if (tp->repair_queue == TCP_NO_QUEUE) 1583 goto out; 1584 1585 /* 'common' recv queue MSG_PEEK-ing */ 1586 } 1587 1588 seq = &tp->copied_seq; 1589 if (flags & MSG_PEEK) { 1590 peek_seq = tp->copied_seq; 1591 seq = &peek_seq; 1592 } 1593 1594 target = sock_rcvlowat(sk, flags & MSG_WAITALL, len); 1595 1596 #ifdef CONFIG_NET_DMA 1597 tp->ucopy.dma_chan = NULL; 1598 preempt_disable(); 1599 skb = skb_peek_tail(&sk->sk_receive_queue); 1600 { 1601 int available = 0; 1602 1603 if (skb) 1604 available = TCP_SKB_CB(skb)->seq + skb->len - (*seq); 1605 if ((available < target) && 1606 (len > sysctl_tcp_dma_copybreak) && !(flags & MSG_PEEK) && 1607 !sysctl_tcp_low_latency && 1608 net_dma_find_channel()) { 1609 preempt_enable_no_resched(); 1610 tp->ucopy.pinned_list = 1611 dma_pin_iovec_pages(msg->msg_iov, len); 1612 } else { 1613 preempt_enable_no_resched(); 1614 } 1615 } 1616 #endif 1617 1618 do { 1619 u32 offset; 1620 1621 /* Are we at urgent data? Stop if we have read anything or have SIGURG pending. */ 1622 if (tp->urg_data && tp->urg_seq == *seq) { 1623 if (copied) 1624 break; 1625 if (signal_pending(current)) { 1626 copied = timeo ? sock_intr_errno(timeo) : -EAGAIN; 1627 break; 1628 } 1629 } 1630 1631 /* Next get a buffer. */ 1632 1633 skb_queue_walk(&sk->sk_receive_queue, skb) { 1634 /* Now that we have two receive queues this 1635 * shouldn't happen. 1636 */ 1637 if (WARN(before(*seq, TCP_SKB_CB(skb)->seq), 1638 "recvmsg bug: copied %X seq %X rcvnxt %X fl %X\n", 1639 *seq, TCP_SKB_CB(skb)->seq, tp->rcv_nxt, 1640 flags)) 1641 break; 1642 1643 offset = *seq - TCP_SKB_CB(skb)->seq; 1644 if (tcp_hdr(skb)->syn) 1645 offset--; 1646 if (offset < skb->len) 1647 goto found_ok_skb; 1648 if (tcp_hdr(skb)->fin) 1649 goto found_fin_ok; 1650 WARN(!(flags & MSG_PEEK), 1651 "recvmsg bug 2: copied %X seq %X rcvnxt %X fl %X\n", 1652 *seq, TCP_SKB_CB(skb)->seq, tp->rcv_nxt, flags); 1653 } 1654 1655 /* Well, if we have backlog, try to process it now yet. */ 1656 1657 if (copied >= target && !sk->sk_backlog.tail) 1658 break; 1659 1660 if (copied) { 1661 if (sk->sk_err || 1662 sk->sk_state == TCP_CLOSE || 1663 (sk->sk_shutdown & RCV_SHUTDOWN) || 1664 !timeo || 1665 signal_pending(current)) 1666 break; 1667 } else { 1668 if (sock_flag(sk, SOCK_DONE)) 1669 break; 1670 1671 if (sk->sk_err) { 1672 copied = sock_error(sk); 1673 break; 1674 } 1675 1676 if (sk->sk_shutdown & RCV_SHUTDOWN) 1677 break; 1678 1679 if (sk->sk_state == TCP_CLOSE) { 1680 if (!sock_flag(sk, SOCK_DONE)) { 1681 /* This occurs when user tries to read 1682 * from never connected socket. 1683 */ 1684 copied = -ENOTCONN; 1685 break; 1686 } 1687 break; 1688 } 1689 1690 if (!timeo) { 1691 copied = -EAGAIN; 1692 break; 1693 } 1694 1695 if (signal_pending(current)) { 1696 copied = sock_intr_errno(timeo); 1697 break; 1698 } 1699 } 1700 1701 tcp_cleanup_rbuf(sk, copied); 1702 1703 if (!sysctl_tcp_low_latency && tp->ucopy.task == user_recv) { 1704 /* Install new reader */ 1705 if (!user_recv && !(flags & (MSG_TRUNC | MSG_PEEK))) { 1706 user_recv = current; 1707 tp->ucopy.task = user_recv; 1708 tp->ucopy.iov = msg->msg_iov; 1709 } 1710 1711 tp->ucopy.len = len; 1712 1713 WARN_ON(tp->copied_seq != tp->rcv_nxt && 1714 !(flags & (MSG_PEEK | MSG_TRUNC))); 1715 1716 /* Ugly... If prequeue is not empty, we have to 1717 * process it before releasing socket, otherwise 1718 * order will be broken at second iteration. 1719 * More elegant solution is required!!! 1720 * 1721 * Look: we have the following (pseudo)queues: 1722 * 1723 * 1. packets in flight 1724 * 2. backlog 1725 * 3. prequeue 1726 * 4. receive_queue 1727 * 1728 * Each queue can be processed only if the next ones 1729 * are empty. At this point we have empty receive_queue. 1730 * But prequeue _can_ be not empty after 2nd iteration, 1731 * when we jumped to start of loop because backlog 1732 * processing added something to receive_queue. 1733 * We cannot release_sock(), because backlog contains 1734 * packets arrived _after_ prequeued ones. 1735 * 1736 * Shortly, algorithm is clear --- to process all 1737 * the queues in order. We could make it more directly, 1738 * requeueing packets from backlog to prequeue, if 1739 * is not empty. It is more elegant, but eats cycles, 1740 * unfortunately. 1741 */ 1742 if (!skb_queue_empty(&tp->ucopy.prequeue)) 1743 goto do_prequeue; 1744 1745 /* __ Set realtime policy in scheduler __ */ 1746 } 1747 1748 #ifdef CONFIG_NET_DMA 1749 if (tp->ucopy.dma_chan) { 1750 if (tp->rcv_wnd == 0 && 1751 !skb_queue_empty(&sk->sk_async_wait_queue)) { 1752 tcp_service_net_dma(sk, true); 1753 tcp_cleanup_rbuf(sk, copied); 1754 } else 1755 dma_async_issue_pending(tp->ucopy.dma_chan); 1756 } 1757 #endif 1758 if (copied >= target) { 1759 /* Do not sleep, just process backlog. */ 1760 release_sock(sk); 1761 lock_sock(sk); 1762 } else 1763 sk_wait_data(sk, &timeo); 1764 1765 #ifdef CONFIG_NET_DMA 1766 tcp_service_net_dma(sk, false); /* Don't block */ 1767 tp->ucopy.wakeup = 0; 1768 #endif 1769 1770 if (user_recv) { 1771 int chunk; 1772 1773 /* __ Restore normal policy in scheduler __ */ 1774 1775 if ((chunk = len - tp->ucopy.len) != 0) { 1776 NET_ADD_STATS_USER(sock_net(sk), LINUX_MIB_TCPDIRECTCOPYFROMBACKLOG, chunk); 1777 len -= chunk; 1778 copied += chunk; 1779 } 1780 1781 if (tp->rcv_nxt == tp->copied_seq && 1782 !skb_queue_empty(&tp->ucopy.prequeue)) { 1783 do_prequeue: 1784 tcp_prequeue_process(sk); 1785 1786 if ((chunk = len - tp->ucopy.len) != 0) { 1787 NET_ADD_STATS_USER(sock_net(sk), LINUX_MIB_TCPDIRECTCOPYFROMPREQUEUE, chunk); 1788 len -= chunk; 1789 copied += chunk; 1790 } 1791 } 1792 } 1793 if ((flags & MSG_PEEK) && 1794 (peek_seq - copied - urg_hole != tp->copied_seq)) { 1795 net_dbg_ratelimited("TCP(%s:%d): Application bug, race in MSG_PEEK\n", 1796 current->comm, 1797 task_pid_nr(current)); 1798 peek_seq = tp->copied_seq; 1799 } 1800 continue; 1801 1802 found_ok_skb: 1803 /* Ok so how much can we use? */ 1804 used = skb->len - offset; 1805 if (len < used) 1806 used = len; 1807 1808 /* Do we have urgent data here? */ 1809 if (tp->urg_data) { 1810 u32 urg_offset = tp->urg_seq - *seq; 1811 if (urg_offset < used) { 1812 if (!urg_offset) { 1813 if (!sock_flag(sk, SOCK_URGINLINE)) { 1814 ++*seq; 1815 urg_hole++; 1816 offset++; 1817 used--; 1818 if (!used) 1819 goto skip_copy; 1820 } 1821 } else 1822 used = urg_offset; 1823 } 1824 } 1825 1826 if (!(flags & MSG_TRUNC)) { 1827 #ifdef CONFIG_NET_DMA 1828 if (!tp->ucopy.dma_chan && tp->ucopy.pinned_list) 1829 tp->ucopy.dma_chan = net_dma_find_channel(); 1830 1831 if (tp->ucopy.dma_chan) { 1832 tp->ucopy.dma_cookie = dma_skb_copy_datagram_iovec( 1833 tp->ucopy.dma_chan, skb, offset, 1834 msg->msg_iov, used, 1835 tp->ucopy.pinned_list); 1836 1837 if (tp->ucopy.dma_cookie < 0) { 1838 1839 pr_alert("%s: dma_cookie < 0\n", 1840 __func__); 1841 1842 /* Exception. Bailout! */ 1843 if (!copied) 1844 copied = -EFAULT; 1845 break; 1846 } 1847 1848 dma_async_issue_pending(tp->ucopy.dma_chan); 1849 1850 if ((offset + used) == skb->len) 1851 copied_early = true; 1852 1853 } else 1854 #endif 1855 { 1856 err = skb_copy_datagram_iovec(skb, offset, 1857 msg->msg_iov, used); 1858 if (err) { 1859 /* Exception. Bailout! */ 1860 if (!copied) 1861 copied = -EFAULT; 1862 break; 1863 } 1864 } 1865 } 1866 1867 *seq += used; 1868 copied += used; 1869 len -= used; 1870 1871 tcp_rcv_space_adjust(sk); 1872 1873 skip_copy: 1874 if (tp->urg_data && after(tp->copied_seq, tp->urg_seq)) { 1875 tp->urg_data = 0; 1876 tcp_fast_path_check(sk); 1877 } 1878 if (used + offset < skb->len) 1879 continue; 1880 1881 if (tcp_hdr(skb)->fin) 1882 goto found_fin_ok; 1883 if (!(flags & MSG_PEEK)) { 1884 sk_eat_skb(sk, skb, copied_early); 1885 copied_early = false; 1886 } 1887 continue; 1888 1889 found_fin_ok: 1890 /* Process the FIN. */ 1891 ++*seq; 1892 if (!(flags & MSG_PEEK)) { 1893 sk_eat_skb(sk, skb, copied_early); 1894 copied_early = false; 1895 } 1896 break; 1897 } while (len > 0); 1898 1899 if (user_recv) { 1900 if (!skb_queue_empty(&tp->ucopy.prequeue)) { 1901 int chunk; 1902 1903 tp->ucopy.len = copied > 0 ? len : 0; 1904 1905 tcp_prequeue_process(sk); 1906 1907 if (copied > 0 && (chunk = len - tp->ucopy.len) != 0) { 1908 NET_ADD_STATS_USER(sock_net(sk), LINUX_MIB_TCPDIRECTCOPYFROMPREQUEUE, chunk); 1909 len -= chunk; 1910 copied += chunk; 1911 } 1912 } 1913 1914 tp->ucopy.task = NULL; 1915 tp->ucopy.len = 0; 1916 } 1917 1918 #ifdef CONFIG_NET_DMA 1919 tcp_service_net_dma(sk, true); /* Wait for queue to drain */ 1920 tp->ucopy.dma_chan = NULL; 1921 1922 if (tp->ucopy.pinned_list) { 1923 dma_unpin_iovec_pages(tp->ucopy.pinned_list); 1924 tp->ucopy.pinned_list = NULL; 1925 } 1926 #endif 1927 1928 /* According to UNIX98, msg_name/msg_namelen are ignored 1929 * on connected socket. I was just happy when found this 8) --ANK 1930 */ 1931 1932 /* Clean up data we have read: This will do ACK frames. */ 1933 tcp_cleanup_rbuf(sk, copied); 1934 1935 release_sock(sk); 1936 return copied; 1937 1938 out: 1939 release_sock(sk); 1940 return err; 1941 1942 recv_urg: 1943 err = tcp_recv_urg(sk, msg, len, flags); 1944 goto out; 1945 1946 recv_sndq: 1947 err = tcp_peek_sndq(sk, msg, len); 1948 goto out; 1949 } 1950 EXPORT_SYMBOL(tcp_recvmsg); 1951 1952 void tcp_set_state(struct sock *sk, int state) 1953 { 1954 int oldstate = sk->sk_state; 1955 1956 switch (state) { 1957 case TCP_ESTABLISHED: 1958 if (oldstate != TCP_ESTABLISHED) 1959 TCP_INC_STATS(sock_net(sk), TCP_MIB_CURRESTAB); 1960 break; 1961 1962 case TCP_CLOSE: 1963 if (oldstate == TCP_CLOSE_WAIT || oldstate == TCP_ESTABLISHED) 1964 TCP_INC_STATS(sock_net(sk), TCP_MIB_ESTABRESETS); 1965 1966 sk->sk_prot->unhash(sk); 1967 if (inet_csk(sk)->icsk_bind_hash && 1968 !(sk->sk_userlocks & SOCK_BINDPORT_LOCK)) 1969 inet_put_port(sk); 1970 /* fall through */ 1971 default: 1972 if (oldstate == TCP_ESTABLISHED) 1973 TCP_DEC_STATS(sock_net(sk), TCP_MIB_CURRESTAB); 1974 } 1975 1976 /* Change state AFTER socket is unhashed to avoid closed 1977 * socket sitting in hash tables. 1978 */ 1979 sk->sk_state = state; 1980 1981 #ifdef STATE_TRACE 1982 SOCK_DEBUG(sk, "TCP sk=%p, State %s -> %s\n", sk, statename[oldstate], statename[state]); 1983 #endif 1984 } 1985 EXPORT_SYMBOL_GPL(tcp_set_state); 1986 1987 /* 1988 * State processing on a close. This implements the state shift for 1989 * sending our FIN frame. Note that we only send a FIN for some 1990 * states. A shutdown() may have already sent the FIN, or we may be 1991 * closed. 1992 */ 1993 1994 static const unsigned char new_state[16] = { 1995 /* current state: new state: action: */ 1996 /* (Invalid) */ TCP_CLOSE, 1997 /* TCP_ESTABLISHED */ TCP_FIN_WAIT1 | TCP_ACTION_FIN, 1998 /* TCP_SYN_SENT */ TCP_CLOSE, 1999 /* TCP_SYN_RECV */ TCP_FIN_WAIT1 | TCP_ACTION_FIN, 2000 /* TCP_FIN_WAIT1 */ TCP_FIN_WAIT1, 2001 /* TCP_FIN_WAIT2 */ TCP_FIN_WAIT2, 2002 /* TCP_TIME_WAIT */ TCP_CLOSE, 2003 /* TCP_CLOSE */ TCP_CLOSE, 2004 /* TCP_CLOSE_WAIT */ TCP_LAST_ACK | TCP_ACTION_FIN, 2005 /* TCP_LAST_ACK */ TCP_LAST_ACK, 2006 /* TCP_LISTEN */ TCP_CLOSE, 2007 /* TCP_CLOSING */ TCP_CLOSING, 2008 }; 2009 2010 static int tcp_close_state(struct sock *sk) 2011 { 2012 int next = (int)new_state[sk->sk_state]; 2013 int ns = next & TCP_STATE_MASK; 2014 2015 tcp_set_state(sk, ns); 2016 2017 return next & TCP_ACTION_FIN; 2018 } 2019 2020 /* 2021 * Shutdown the sending side of a connection. Much like close except 2022 * that we don't receive shut down or sock_set_flag(sk, SOCK_DEAD). 2023 */ 2024 2025 void tcp_shutdown(struct sock *sk, int how) 2026 { 2027 /* We need to grab some memory, and put together a FIN, 2028 * and then put it into the queue to be sent. 2029 * Tim MacKenzie(tym@dibbler.cs.monash.edu.au) 4 Dec '92. 2030 */ 2031 if (!(how & SEND_SHUTDOWN)) 2032 return; 2033 2034 /* If we've already sent a FIN, or it's a closed state, skip this. */ 2035 if ((1 << sk->sk_state) & 2036 (TCPF_ESTABLISHED | TCPF_SYN_SENT | 2037 TCPF_SYN_RECV | TCPF_CLOSE_WAIT)) { 2038 /* Clear out any half completed packets. FIN if needed. */ 2039 if (tcp_close_state(sk)) 2040 tcp_send_fin(sk); 2041 } 2042 } 2043 EXPORT_SYMBOL(tcp_shutdown); 2044 2045 bool tcp_check_oom(struct sock *sk, int shift) 2046 { 2047 bool too_many_orphans, out_of_socket_memory; 2048 2049 too_many_orphans = tcp_too_many_orphans(sk, shift); 2050 out_of_socket_memory = tcp_out_of_memory(sk); 2051 2052 if (too_many_orphans) 2053 net_info_ratelimited("too many orphaned sockets\n"); 2054 if (out_of_socket_memory) 2055 net_info_ratelimited("out of memory -- consider tuning tcp_mem\n"); 2056 return too_many_orphans || out_of_socket_memory; 2057 } 2058 2059 void tcp_close(struct sock *sk, long timeout) 2060 { 2061 struct sk_buff *skb; 2062 int data_was_unread = 0; 2063 int state; 2064 2065 lock_sock(sk); 2066 sk->sk_shutdown = SHUTDOWN_MASK; 2067 2068 if (sk->sk_state == TCP_LISTEN) { 2069 tcp_set_state(sk, TCP_CLOSE); 2070 2071 /* Special case. */ 2072 inet_csk_listen_stop(sk); 2073 2074 goto adjudge_to_death; 2075 } 2076 2077 /* We need to flush the recv. buffs. We do this only on the 2078 * descriptor close, not protocol-sourced closes, because the 2079 * reader process may not have drained the data yet! 2080 */ 2081 while ((skb = __skb_dequeue(&sk->sk_receive_queue)) != NULL) { 2082 u32 len = TCP_SKB_CB(skb)->end_seq - TCP_SKB_CB(skb)->seq - 2083 tcp_hdr(skb)->fin; 2084 data_was_unread += len; 2085 __kfree_skb(skb); 2086 } 2087 2088 sk_mem_reclaim(sk); 2089 2090 /* If socket has been already reset (e.g. in tcp_reset()) - kill it. */ 2091 if (sk->sk_state == TCP_CLOSE) 2092 goto adjudge_to_death; 2093 2094 /* As outlined in RFC 2525, section 2.17, we send a RST here because 2095 * data was lost. To witness the awful effects of the old behavior of 2096 * always doing a FIN, run an older 2.1.x kernel or 2.0.x, start a bulk 2097 * GET in an FTP client, suspend the process, wait for the client to 2098 * advertise a zero window, then kill -9 the FTP client, wheee... 2099 * Note: timeout is always zero in such a case. 2100 */ 2101 if (unlikely(tcp_sk(sk)->repair)) { 2102 sk->sk_prot->disconnect(sk, 0); 2103 } else if (data_was_unread) { 2104 /* Unread data was tossed, zap the connection. */ 2105 NET_INC_STATS_USER(sock_net(sk), LINUX_MIB_TCPABORTONCLOSE); 2106 tcp_set_state(sk, TCP_CLOSE); 2107 tcp_send_active_reset(sk, sk->sk_allocation); 2108 } else if (sock_flag(sk, SOCK_LINGER) && !sk->sk_lingertime) { 2109 /* Check zero linger _after_ checking for unread data. */ 2110 sk->sk_prot->disconnect(sk, 0); 2111 NET_INC_STATS_USER(sock_net(sk), LINUX_MIB_TCPABORTONDATA); 2112 } else if (tcp_close_state(sk)) { 2113 /* We FIN if the application ate all the data before 2114 * zapping the connection. 2115 */ 2116 2117 /* RED-PEN. Formally speaking, we have broken TCP state 2118 * machine. State transitions: 2119 * 2120 * TCP_ESTABLISHED -> TCP_FIN_WAIT1 2121 * TCP_SYN_RECV -> TCP_FIN_WAIT1 (forget it, it's impossible) 2122 * TCP_CLOSE_WAIT -> TCP_LAST_ACK 2123 * 2124 * are legal only when FIN has been sent (i.e. in window), 2125 * rather than queued out of window. Purists blame. 2126 * 2127 * F.e. "RFC state" is ESTABLISHED, 2128 * if Linux state is FIN-WAIT-1, but FIN is still not sent. 2129 * 2130 * The visible declinations are that sometimes 2131 * we enter time-wait state, when it is not required really 2132 * (harmless), do not send active resets, when they are 2133 * required by specs (TCP_ESTABLISHED, TCP_CLOSE_WAIT, when 2134 * they look as CLOSING or LAST_ACK for Linux) 2135 * Probably, I missed some more holelets. 2136 * --ANK 2137 * XXX (TFO) - To start off we don't support SYN+ACK+FIN 2138 * in a single packet! (May consider it later but will 2139 * probably need API support or TCP_CORK SYN-ACK until 2140 * data is written and socket is closed.) 2141 */ 2142 tcp_send_fin(sk); 2143 } 2144 2145 sk_stream_wait_close(sk, timeout); 2146 2147 adjudge_to_death: 2148 state = sk->sk_state; 2149 sock_hold(sk); 2150 sock_orphan(sk); 2151 2152 /* It is the last release_sock in its life. It will remove backlog. */ 2153 release_sock(sk); 2154 2155 2156 /* Now socket is owned by kernel and we acquire BH lock 2157 to finish close. No need to check for user refs. 2158 */ 2159 local_bh_disable(); 2160 bh_lock_sock(sk); 2161 WARN_ON(sock_owned_by_user(sk)); 2162 2163 percpu_counter_inc(sk->sk_prot->orphan_count); 2164 2165 /* Have we already been destroyed by a softirq or backlog? */ 2166 if (state != TCP_CLOSE && sk->sk_state == TCP_CLOSE) 2167 goto out; 2168 2169 /* This is a (useful) BSD violating of the RFC. There is a 2170 * problem with TCP as specified in that the other end could 2171 * keep a socket open forever with no application left this end. 2172 * We use a 3 minute timeout (about the same as BSD) then kill 2173 * our end. If they send after that then tough - BUT: long enough 2174 * that we won't make the old 4*rto = almost no time - whoops 2175 * reset mistake. 2176 * 2177 * Nope, it was not mistake. It is really desired behaviour 2178 * f.e. on http servers, when such sockets are useless, but 2179 * consume significant resources. Let's do it with special 2180 * linger2 option. --ANK 2181 */ 2182 2183 if (sk->sk_state == TCP_FIN_WAIT2) { 2184 struct tcp_sock *tp = tcp_sk(sk); 2185 if (tp->linger2 < 0) { 2186 tcp_set_state(sk, TCP_CLOSE); 2187 tcp_send_active_reset(sk, GFP_ATOMIC); 2188 NET_INC_STATS_BH(sock_net(sk), 2189 LINUX_MIB_TCPABORTONLINGER); 2190 } else { 2191 const int tmo = tcp_fin_time(sk); 2192 2193 if (tmo > TCP_TIMEWAIT_LEN) { 2194 inet_csk_reset_keepalive_timer(sk, 2195 tmo - TCP_TIMEWAIT_LEN); 2196 } else { 2197 tcp_time_wait(sk, TCP_FIN_WAIT2, tmo); 2198 goto out; 2199 } 2200 } 2201 } 2202 if (sk->sk_state != TCP_CLOSE) { 2203 sk_mem_reclaim(sk); 2204 if (tcp_check_oom(sk, 0)) { 2205 tcp_set_state(sk, TCP_CLOSE); 2206 tcp_send_active_reset(sk, GFP_ATOMIC); 2207 NET_INC_STATS_BH(sock_net(sk), 2208 LINUX_MIB_TCPABORTONMEMORY); 2209 } 2210 } 2211 2212 if (sk->sk_state == TCP_CLOSE) { 2213 struct request_sock *req = tcp_sk(sk)->fastopen_rsk; 2214 /* We could get here with a non-NULL req if the socket is 2215 * aborted (e.g., closed with unread data) before 3WHS 2216 * finishes. 2217 */ 2218 if (req != NULL) 2219 reqsk_fastopen_remove(sk, req, false); 2220 inet_csk_destroy_sock(sk); 2221 } 2222 /* Otherwise, socket is reprieved until protocol close. */ 2223 2224 out: 2225 bh_unlock_sock(sk); 2226 local_bh_enable(); 2227 sock_put(sk); 2228 } 2229 EXPORT_SYMBOL(tcp_close); 2230 2231 /* These states need RST on ABORT according to RFC793 */ 2232 2233 static inline bool tcp_need_reset(int state) 2234 { 2235 return (1 << state) & 2236 (TCPF_ESTABLISHED | TCPF_CLOSE_WAIT | TCPF_FIN_WAIT1 | 2237 TCPF_FIN_WAIT2 | TCPF_SYN_RECV); 2238 } 2239 2240 int tcp_disconnect(struct sock *sk, int flags) 2241 { 2242 struct inet_sock *inet = inet_sk(sk); 2243 struct inet_connection_sock *icsk = inet_csk(sk); 2244 struct tcp_sock *tp = tcp_sk(sk); 2245 int err = 0; 2246 int old_state = sk->sk_state; 2247 2248 if (old_state != TCP_CLOSE) 2249 tcp_set_state(sk, TCP_CLOSE); 2250 2251 /* ABORT function of RFC793 */ 2252 if (old_state == TCP_LISTEN) { 2253 inet_csk_listen_stop(sk); 2254 } else if (unlikely(tp->repair)) { 2255 sk->sk_err = ECONNABORTED; 2256 } else if (tcp_need_reset(old_state) || 2257 (tp->snd_nxt != tp->write_seq && 2258 (1 << old_state) & (TCPF_CLOSING | TCPF_LAST_ACK))) { 2259 /* The last check adjusts for discrepancy of Linux wrt. RFC 2260 * states 2261 */ 2262 tcp_send_active_reset(sk, gfp_any()); 2263 sk->sk_err = ECONNRESET; 2264 } else if (old_state == TCP_SYN_SENT) 2265 sk->sk_err = ECONNRESET; 2266 2267 tcp_clear_xmit_timers(sk); 2268 __skb_queue_purge(&sk->sk_receive_queue); 2269 tcp_write_queue_purge(sk); 2270 __skb_queue_purge(&tp->out_of_order_queue); 2271 #ifdef CONFIG_NET_DMA 2272 __skb_queue_purge(&sk->sk_async_wait_queue); 2273 #endif 2274 2275 inet->inet_dport = 0; 2276 2277 if (!(sk->sk_userlocks & SOCK_BINDADDR_LOCK)) 2278 inet_reset_saddr(sk); 2279 2280 sk->sk_shutdown = 0; 2281 sock_reset_flag(sk, SOCK_DONE); 2282 tp->srtt = 0; 2283 if ((tp->write_seq += tp->max_window + 2) == 0) 2284 tp->write_seq = 1; 2285 icsk->icsk_backoff = 0; 2286 tp->snd_cwnd = 2; 2287 icsk->icsk_probes_out = 0; 2288 tp->packets_out = 0; 2289 tp->snd_ssthresh = TCP_INFINITE_SSTHRESH; 2290 tp->snd_cwnd_cnt = 0; 2291 tp->window_clamp = 0; 2292 tcp_set_ca_state(sk, TCP_CA_Open); 2293 tcp_clear_retrans(tp); 2294 inet_csk_delack_init(sk); 2295 tcp_init_send_head(sk); 2296 memset(&tp->rx_opt, 0, sizeof(tp->rx_opt)); 2297 __sk_dst_reset(sk); 2298 2299 WARN_ON(inet->inet_num && !icsk->icsk_bind_hash); 2300 2301 sk->sk_error_report(sk); 2302 return err; 2303 } 2304 EXPORT_SYMBOL(tcp_disconnect); 2305 2306 void tcp_sock_destruct(struct sock *sk) 2307 { 2308 inet_sock_destruct(sk); 2309 2310 kfree(inet_csk(sk)->icsk_accept_queue.fastopenq); 2311 } 2312 2313 static inline bool tcp_can_repair_sock(const struct sock *sk) 2314 { 2315 return ns_capable(sock_net(sk)->user_ns, CAP_NET_ADMIN) && 2316 ((1 << sk->sk_state) & (TCPF_CLOSE | TCPF_ESTABLISHED)); 2317 } 2318 2319 static int tcp_repair_options_est(struct tcp_sock *tp, 2320 struct tcp_repair_opt __user *optbuf, unsigned int len) 2321 { 2322 struct tcp_repair_opt opt; 2323 2324 while (len >= sizeof(opt)) { 2325 if (copy_from_user(&opt, optbuf, sizeof(opt))) 2326 return -EFAULT; 2327 2328 optbuf++; 2329 len -= sizeof(opt); 2330 2331 switch (opt.opt_code) { 2332 case TCPOPT_MSS: 2333 tp->rx_opt.mss_clamp = opt.opt_val; 2334 break; 2335 case TCPOPT_WINDOW: 2336 { 2337 u16 snd_wscale = opt.opt_val & 0xFFFF; 2338 u16 rcv_wscale = opt.opt_val >> 16; 2339 2340 if (snd_wscale > 14 || rcv_wscale > 14) 2341 return -EFBIG; 2342 2343 tp->rx_opt.snd_wscale = snd_wscale; 2344 tp->rx_opt.rcv_wscale = rcv_wscale; 2345 tp->rx_opt.wscale_ok = 1; 2346 } 2347 break; 2348 case TCPOPT_SACK_PERM: 2349 if (opt.opt_val != 0) 2350 return -EINVAL; 2351 2352 tp->rx_opt.sack_ok |= TCP_SACK_SEEN; 2353 if (sysctl_tcp_fack) 2354 tcp_enable_fack(tp); 2355 break; 2356 case TCPOPT_TIMESTAMP: 2357 if (opt.opt_val != 0) 2358 return -EINVAL; 2359 2360 tp->rx_opt.tstamp_ok = 1; 2361 break; 2362 } 2363 } 2364 2365 return 0; 2366 } 2367 2368 /* 2369 * Socket option code for TCP. 2370 */ 2371 static int do_tcp_setsockopt(struct sock *sk, int level, 2372 int optname, char __user *optval, unsigned int optlen) 2373 { 2374 struct tcp_sock *tp = tcp_sk(sk); 2375 struct inet_connection_sock *icsk = inet_csk(sk); 2376 int val; 2377 int err = 0; 2378 2379 /* These are data/string values, all the others are ints */ 2380 switch (optname) { 2381 case TCP_CONGESTION: { 2382 char name[TCP_CA_NAME_MAX]; 2383 2384 if (optlen < 1) 2385 return -EINVAL; 2386 2387 val = strncpy_from_user(name, optval, 2388 min_t(long, TCP_CA_NAME_MAX-1, optlen)); 2389 if (val < 0) 2390 return -EFAULT; 2391 name[val] = 0; 2392 2393 lock_sock(sk); 2394 err = tcp_set_congestion_control(sk, name); 2395 release_sock(sk); 2396 return err; 2397 } 2398 default: 2399 /* fallthru */ 2400 break; 2401 } 2402 2403 if (optlen < sizeof(int)) 2404 return -EINVAL; 2405 2406 if (get_user(val, (int __user *)optval)) 2407 return -EFAULT; 2408 2409 lock_sock(sk); 2410 2411 switch (optname) { 2412 case TCP_MAXSEG: 2413 /* Values greater than interface MTU won't take effect. However 2414 * at the point when this call is done we typically don't yet 2415 * know which interface is going to be used */ 2416 if (val < TCP_MIN_MSS || val > MAX_TCP_WINDOW) { 2417 err = -EINVAL; 2418 break; 2419 } 2420 tp->rx_opt.user_mss = val; 2421 break; 2422 2423 case TCP_NODELAY: 2424 if (val) { 2425 /* TCP_NODELAY is weaker than TCP_CORK, so that 2426 * this option on corked socket is remembered, but 2427 * it is not activated until cork is cleared. 2428 * 2429 * However, when TCP_NODELAY is set we make 2430 * an explicit push, which overrides even TCP_CORK 2431 * for currently queued segments. 2432 */ 2433 tp->nonagle |= TCP_NAGLE_OFF|TCP_NAGLE_PUSH; 2434 tcp_push_pending_frames(sk); 2435 } else { 2436 tp->nonagle &= ~TCP_NAGLE_OFF; 2437 } 2438 break; 2439 2440 case TCP_THIN_LINEAR_TIMEOUTS: 2441 if (val < 0 || val > 1) 2442 err = -EINVAL; 2443 else 2444 tp->thin_lto = val; 2445 break; 2446 2447 case TCP_THIN_DUPACK: 2448 if (val < 0 || val > 1) 2449 err = -EINVAL; 2450 else 2451 tp->thin_dupack = val; 2452 if (tp->thin_dupack) 2453 tcp_disable_early_retrans(tp); 2454 break; 2455 2456 case TCP_REPAIR: 2457 if (!tcp_can_repair_sock(sk)) 2458 err = -EPERM; 2459 else if (val == 1) { 2460 tp->repair = 1; 2461 sk->sk_reuse = SK_FORCE_REUSE; 2462 tp->repair_queue = TCP_NO_QUEUE; 2463 } else if (val == 0) { 2464 tp->repair = 0; 2465 sk->sk_reuse = SK_NO_REUSE; 2466 tcp_send_window_probe(sk); 2467 } else 2468 err = -EINVAL; 2469 2470 break; 2471 2472 case TCP_REPAIR_QUEUE: 2473 if (!tp->repair) 2474 err = -EPERM; 2475 else if (val < TCP_QUEUES_NR) 2476 tp->repair_queue = val; 2477 else 2478 err = -EINVAL; 2479 break; 2480 2481 case TCP_QUEUE_SEQ: 2482 if (sk->sk_state != TCP_CLOSE) 2483 err = -EPERM; 2484 else if (tp->repair_queue == TCP_SEND_QUEUE) 2485 tp->write_seq = val; 2486 else if (tp->repair_queue == TCP_RECV_QUEUE) 2487 tp->rcv_nxt = val; 2488 else 2489 err = -EINVAL; 2490 break; 2491 2492 case TCP_REPAIR_OPTIONS: 2493 if (!tp->repair) 2494 err = -EINVAL; 2495 else if (sk->sk_state == TCP_ESTABLISHED) 2496 err = tcp_repair_options_est(tp, 2497 (struct tcp_repair_opt __user *)optval, 2498 optlen); 2499 else 2500 err = -EPERM; 2501 break; 2502 2503 case TCP_CORK: 2504 /* When set indicates to always queue non-full frames. 2505 * Later the user clears this option and we transmit 2506 * any pending partial frames in the queue. This is 2507 * meant to be used alongside sendfile() to get properly 2508 * filled frames when the user (for example) must write 2509 * out headers with a write() call first and then use 2510 * sendfile to send out the data parts. 2511 * 2512 * TCP_CORK can be set together with TCP_NODELAY and it is 2513 * stronger than TCP_NODELAY. 2514 */ 2515 if (val) { 2516 tp->nonagle |= TCP_NAGLE_CORK; 2517 } else { 2518 tp->nonagle &= ~TCP_NAGLE_CORK; 2519 if (tp->nonagle&TCP_NAGLE_OFF) 2520 tp->nonagle |= TCP_NAGLE_PUSH; 2521 tcp_push_pending_frames(sk); 2522 } 2523 break; 2524 2525 case TCP_KEEPIDLE: 2526 if (val < 1 || val > MAX_TCP_KEEPIDLE) 2527 err = -EINVAL; 2528 else { 2529 tp->keepalive_time = val * HZ; 2530 if (sock_flag(sk, SOCK_KEEPOPEN) && 2531 !((1 << sk->sk_state) & 2532 (TCPF_CLOSE | TCPF_LISTEN))) { 2533 u32 elapsed = keepalive_time_elapsed(tp); 2534 if (tp->keepalive_time > elapsed) 2535 elapsed = tp->keepalive_time - elapsed; 2536 else 2537 elapsed = 0; 2538 inet_csk_reset_keepalive_timer(sk, elapsed); 2539 } 2540 } 2541 break; 2542 case TCP_KEEPINTVL: 2543 if (val < 1 || val > MAX_TCP_KEEPINTVL) 2544 err = -EINVAL; 2545 else 2546 tp->keepalive_intvl = val * HZ; 2547 break; 2548 case TCP_KEEPCNT: 2549 if (val < 1 || val > MAX_TCP_KEEPCNT) 2550 err = -EINVAL; 2551 else 2552 tp->keepalive_probes = val; 2553 break; 2554 case TCP_SYNCNT: 2555 if (val < 1 || val > MAX_TCP_SYNCNT) 2556 err = -EINVAL; 2557 else 2558 icsk->icsk_syn_retries = val; 2559 break; 2560 2561 case TCP_LINGER2: 2562 if (val < 0) 2563 tp->linger2 = -1; 2564 else if (val > sysctl_tcp_fin_timeout / HZ) 2565 tp->linger2 = 0; 2566 else 2567 tp->linger2 = val * HZ; 2568 break; 2569 2570 case TCP_DEFER_ACCEPT: 2571 /* Translate value in seconds to number of retransmits */ 2572 icsk->icsk_accept_queue.rskq_defer_accept = 2573 secs_to_retrans(val, TCP_TIMEOUT_INIT / HZ, 2574 TCP_RTO_MAX / HZ); 2575 break; 2576 2577 case TCP_WINDOW_CLAMP: 2578 if (!val) { 2579 if (sk->sk_state != TCP_CLOSE) { 2580 err = -EINVAL; 2581 break; 2582 } 2583 tp->window_clamp = 0; 2584 } else 2585 tp->window_clamp = val < SOCK_MIN_RCVBUF / 2 ? 2586 SOCK_MIN_RCVBUF / 2 : val; 2587 break; 2588 2589 case TCP_QUICKACK: 2590 if (!val) { 2591 icsk->icsk_ack.pingpong = 1; 2592 } else { 2593 icsk->icsk_ack.pingpong = 0; 2594 if ((1 << sk->sk_state) & 2595 (TCPF_ESTABLISHED | TCPF_CLOSE_WAIT) && 2596 inet_csk_ack_scheduled(sk)) { 2597 icsk->icsk_ack.pending |= ICSK_ACK_PUSHED; 2598 tcp_cleanup_rbuf(sk, 1); 2599 if (!(val & 1)) 2600 icsk->icsk_ack.pingpong = 1; 2601 } 2602 } 2603 break; 2604 2605 #ifdef CONFIG_TCP_MD5SIG 2606 case TCP_MD5SIG: 2607 /* Read the IP->Key mappings from userspace */ 2608 err = tp->af_specific->md5_parse(sk, optval, optlen); 2609 break; 2610 #endif 2611 case TCP_USER_TIMEOUT: 2612 /* Cap the max timeout in ms TCP will retry/retrans 2613 * before giving up and aborting (ETIMEDOUT) a connection. 2614 */ 2615 if (val < 0) 2616 err = -EINVAL; 2617 else 2618 icsk->icsk_user_timeout = msecs_to_jiffies(val); 2619 break; 2620 2621 case TCP_FASTOPEN: 2622 if (val >= 0 && ((1 << sk->sk_state) & (TCPF_CLOSE | 2623 TCPF_LISTEN))) 2624 err = fastopen_init_queue(sk, val); 2625 else 2626 err = -EINVAL; 2627 break; 2628 case TCP_TIMESTAMP: 2629 if (!tp->repair) 2630 err = -EPERM; 2631 else 2632 tp->tsoffset = val - tcp_time_stamp; 2633 break; 2634 default: 2635 err = -ENOPROTOOPT; 2636 break; 2637 } 2638 2639 release_sock(sk); 2640 return err; 2641 } 2642 2643 int tcp_setsockopt(struct sock *sk, int level, int optname, char __user *optval, 2644 unsigned int optlen) 2645 { 2646 const struct inet_connection_sock *icsk = inet_csk(sk); 2647 2648 if (level != SOL_TCP) 2649 return icsk->icsk_af_ops->setsockopt(sk, level, optname, 2650 optval, optlen); 2651 return do_tcp_setsockopt(sk, level, optname, optval, optlen); 2652 } 2653 EXPORT_SYMBOL(tcp_setsockopt); 2654 2655 #ifdef CONFIG_COMPAT 2656 int compat_tcp_setsockopt(struct sock *sk, int level, int optname, 2657 char __user *optval, unsigned int optlen) 2658 { 2659 if (level != SOL_TCP) 2660 return inet_csk_compat_setsockopt(sk, level, optname, 2661 optval, optlen); 2662 return do_tcp_setsockopt(sk, level, optname, optval, optlen); 2663 } 2664 EXPORT_SYMBOL(compat_tcp_setsockopt); 2665 #endif 2666 2667 /* Return information about state of tcp endpoint in API format. */ 2668 void tcp_get_info(const struct sock *sk, struct tcp_info *info) 2669 { 2670 const struct tcp_sock *tp = tcp_sk(sk); 2671 const struct inet_connection_sock *icsk = inet_csk(sk); 2672 u32 now = tcp_time_stamp; 2673 2674 memset(info, 0, sizeof(*info)); 2675 2676 info->tcpi_state = sk->sk_state; 2677 info->tcpi_ca_state = icsk->icsk_ca_state; 2678 info->tcpi_retransmits = icsk->icsk_retransmits; 2679 info->tcpi_probes = icsk->icsk_probes_out; 2680 info->tcpi_backoff = icsk->icsk_backoff; 2681 2682 if (tp->rx_opt.tstamp_ok) 2683 info->tcpi_options |= TCPI_OPT_TIMESTAMPS; 2684 if (tcp_is_sack(tp)) 2685 info->tcpi_options |= TCPI_OPT_SACK; 2686 if (tp->rx_opt.wscale_ok) { 2687 info->tcpi_options |= TCPI_OPT_WSCALE; 2688 info->tcpi_snd_wscale = tp->rx_opt.snd_wscale; 2689 info->tcpi_rcv_wscale = tp->rx_opt.rcv_wscale; 2690 } 2691 2692 if (tp->ecn_flags & TCP_ECN_OK) 2693 info->tcpi_options |= TCPI_OPT_ECN; 2694 if (tp->ecn_flags & TCP_ECN_SEEN) 2695 info->tcpi_options |= TCPI_OPT_ECN_SEEN; 2696 if (tp->syn_data_acked) 2697 info->tcpi_options |= TCPI_OPT_SYN_DATA; 2698 2699 info->tcpi_rto = jiffies_to_usecs(icsk->icsk_rto); 2700 info->tcpi_ato = jiffies_to_usecs(icsk->icsk_ack.ato); 2701 info->tcpi_snd_mss = tp->mss_cache; 2702 info->tcpi_rcv_mss = icsk->icsk_ack.rcv_mss; 2703 2704 if (sk->sk_state == TCP_LISTEN) { 2705 info->tcpi_unacked = sk->sk_ack_backlog; 2706 info->tcpi_sacked = sk->sk_max_ack_backlog; 2707 } else { 2708 info->tcpi_unacked = tp->packets_out; 2709 info->tcpi_sacked = tp->sacked_out; 2710 } 2711 info->tcpi_lost = tp->lost_out; 2712 info->tcpi_retrans = tp->retrans_out; 2713 info->tcpi_fackets = tp->fackets_out; 2714 2715 info->tcpi_last_data_sent = jiffies_to_msecs(now - tp->lsndtime); 2716 info->tcpi_last_data_recv = jiffies_to_msecs(now - icsk->icsk_ack.lrcvtime); 2717 info->tcpi_last_ack_recv = jiffies_to_msecs(now - tp->rcv_tstamp); 2718 2719 info->tcpi_pmtu = icsk->icsk_pmtu_cookie; 2720 info->tcpi_rcv_ssthresh = tp->rcv_ssthresh; 2721 info->tcpi_rtt = jiffies_to_usecs(tp->srtt)>>3; 2722 info->tcpi_rttvar = jiffies_to_usecs(tp->mdev)>>2; 2723 info->tcpi_snd_ssthresh = tp->snd_ssthresh; 2724 info->tcpi_snd_cwnd = tp->snd_cwnd; 2725 info->tcpi_advmss = tp->advmss; 2726 info->tcpi_reordering = tp->reordering; 2727 2728 info->tcpi_rcv_rtt = jiffies_to_usecs(tp->rcv_rtt_est.rtt)>>3; 2729 info->tcpi_rcv_space = tp->rcvq_space.space; 2730 2731 info->tcpi_total_retrans = tp->total_retrans; 2732 } 2733 EXPORT_SYMBOL_GPL(tcp_get_info); 2734 2735 static int do_tcp_getsockopt(struct sock *sk, int level, 2736 int optname, char __user *optval, int __user *optlen) 2737 { 2738 struct inet_connection_sock *icsk = inet_csk(sk); 2739 struct tcp_sock *tp = tcp_sk(sk); 2740 int val, len; 2741 2742 if (get_user(len, optlen)) 2743 return -EFAULT; 2744 2745 len = min_t(unsigned int, len, sizeof(int)); 2746 2747 if (len < 0) 2748 return -EINVAL; 2749 2750 switch (optname) { 2751 case TCP_MAXSEG: 2752 val = tp->mss_cache; 2753 if (!val && ((1 << sk->sk_state) & (TCPF_CLOSE | TCPF_LISTEN))) 2754 val = tp->rx_opt.user_mss; 2755 if (tp->repair) 2756 val = tp->rx_opt.mss_clamp; 2757 break; 2758 case TCP_NODELAY: 2759 val = !!(tp->nonagle&TCP_NAGLE_OFF); 2760 break; 2761 case TCP_CORK: 2762 val = !!(tp->nonagle&TCP_NAGLE_CORK); 2763 break; 2764 case TCP_KEEPIDLE: 2765 val = keepalive_time_when(tp) / HZ; 2766 break; 2767 case TCP_KEEPINTVL: 2768 val = keepalive_intvl_when(tp) / HZ; 2769 break; 2770 case TCP_KEEPCNT: 2771 val = keepalive_probes(tp); 2772 break; 2773 case TCP_SYNCNT: 2774 val = icsk->icsk_syn_retries ? : sysctl_tcp_syn_retries; 2775 break; 2776 case TCP_LINGER2: 2777 val = tp->linger2; 2778 if (val >= 0) 2779 val = (val ? : sysctl_tcp_fin_timeout) / HZ; 2780 break; 2781 case TCP_DEFER_ACCEPT: 2782 val = retrans_to_secs(icsk->icsk_accept_queue.rskq_defer_accept, 2783 TCP_TIMEOUT_INIT / HZ, TCP_RTO_MAX / HZ); 2784 break; 2785 case TCP_WINDOW_CLAMP: 2786 val = tp->window_clamp; 2787 break; 2788 case TCP_INFO: { 2789 struct tcp_info info; 2790 2791 if (get_user(len, optlen)) 2792 return -EFAULT; 2793 2794 tcp_get_info(sk, &info); 2795 2796 len = min_t(unsigned int, len, sizeof(info)); 2797 if (put_user(len, optlen)) 2798 return -EFAULT; 2799 if (copy_to_user(optval, &info, len)) 2800 return -EFAULT; 2801 return 0; 2802 } 2803 case TCP_QUICKACK: 2804 val = !icsk->icsk_ack.pingpong; 2805 break; 2806 2807 case TCP_CONGESTION: 2808 if (get_user(len, optlen)) 2809 return -EFAULT; 2810 len = min_t(unsigned int, len, TCP_CA_NAME_MAX); 2811 if (put_user(len, optlen)) 2812 return -EFAULT; 2813 if (copy_to_user(optval, icsk->icsk_ca_ops->name, len)) 2814 return -EFAULT; 2815 return 0; 2816 2817 case TCP_THIN_LINEAR_TIMEOUTS: 2818 val = tp->thin_lto; 2819 break; 2820 case TCP_THIN_DUPACK: 2821 val = tp->thin_dupack; 2822 break; 2823 2824 case TCP_REPAIR: 2825 val = tp->repair; 2826 break; 2827 2828 case TCP_REPAIR_QUEUE: 2829 if (tp->repair) 2830 val = tp->repair_queue; 2831 else 2832 return -EINVAL; 2833 break; 2834 2835 case TCP_QUEUE_SEQ: 2836 if (tp->repair_queue == TCP_SEND_QUEUE) 2837 val = tp->write_seq; 2838 else if (tp->repair_queue == TCP_RECV_QUEUE) 2839 val = tp->rcv_nxt; 2840 else 2841 return -EINVAL; 2842 break; 2843 2844 case TCP_USER_TIMEOUT: 2845 val = jiffies_to_msecs(icsk->icsk_user_timeout); 2846 break; 2847 case TCP_TIMESTAMP: 2848 val = tcp_time_stamp + tp->tsoffset; 2849 break; 2850 default: 2851 return -ENOPROTOOPT; 2852 } 2853 2854 if (put_user(len, optlen)) 2855 return -EFAULT; 2856 if (copy_to_user(optval, &val, len)) 2857 return -EFAULT; 2858 return 0; 2859 } 2860 2861 int tcp_getsockopt(struct sock *sk, int level, int optname, char __user *optval, 2862 int __user *optlen) 2863 { 2864 struct inet_connection_sock *icsk = inet_csk(sk); 2865 2866 if (level != SOL_TCP) 2867 return icsk->icsk_af_ops->getsockopt(sk, level, optname, 2868 optval, optlen); 2869 return do_tcp_getsockopt(sk, level, optname, optval, optlen); 2870 } 2871 EXPORT_SYMBOL(tcp_getsockopt); 2872 2873 #ifdef CONFIG_COMPAT 2874 int compat_tcp_getsockopt(struct sock *sk, int level, int optname, 2875 char __user *optval, int __user *optlen) 2876 { 2877 if (level != SOL_TCP) 2878 return inet_csk_compat_getsockopt(sk, level, optname, 2879 optval, optlen); 2880 return do_tcp_getsockopt(sk, level, optname, optval, optlen); 2881 } 2882 EXPORT_SYMBOL(compat_tcp_getsockopt); 2883 #endif 2884 2885 #ifdef CONFIG_TCP_MD5SIG 2886 static struct tcp_md5sig_pool __percpu *tcp_md5sig_pool __read_mostly; 2887 static DEFINE_MUTEX(tcp_md5sig_mutex); 2888 2889 static void __tcp_free_md5sig_pool(struct tcp_md5sig_pool __percpu *pool) 2890 { 2891 int cpu; 2892 2893 for_each_possible_cpu(cpu) { 2894 struct tcp_md5sig_pool *p = per_cpu_ptr(pool, cpu); 2895 2896 if (p->md5_desc.tfm) 2897 crypto_free_hash(p->md5_desc.tfm); 2898 } 2899 free_percpu(pool); 2900 } 2901 2902 static void __tcp_alloc_md5sig_pool(void) 2903 { 2904 int cpu; 2905 struct tcp_md5sig_pool __percpu *pool; 2906 2907 pool = alloc_percpu(struct tcp_md5sig_pool); 2908 if (!pool) 2909 return; 2910 2911 for_each_possible_cpu(cpu) { 2912 struct crypto_hash *hash; 2913 2914 hash = crypto_alloc_hash("md5", 0, CRYPTO_ALG_ASYNC); 2915 if (IS_ERR_OR_NULL(hash)) 2916 goto out_free; 2917 2918 per_cpu_ptr(pool, cpu)->md5_desc.tfm = hash; 2919 } 2920 /* before setting tcp_md5sig_pool, we must commit all writes 2921 * to memory. See ACCESS_ONCE() in tcp_get_md5sig_pool() 2922 */ 2923 smp_wmb(); 2924 tcp_md5sig_pool = pool; 2925 return; 2926 out_free: 2927 __tcp_free_md5sig_pool(pool); 2928 } 2929 2930 bool tcp_alloc_md5sig_pool(void) 2931 { 2932 if (unlikely(!tcp_md5sig_pool)) { 2933 mutex_lock(&tcp_md5sig_mutex); 2934 2935 if (!tcp_md5sig_pool) 2936 __tcp_alloc_md5sig_pool(); 2937 2938 mutex_unlock(&tcp_md5sig_mutex); 2939 } 2940 return tcp_md5sig_pool != NULL; 2941 } 2942 EXPORT_SYMBOL(tcp_alloc_md5sig_pool); 2943 2944 2945 /** 2946 * tcp_get_md5sig_pool - get md5sig_pool for this user 2947 * 2948 * We use percpu structure, so if we succeed, we exit with preemption 2949 * and BH disabled, to make sure another thread or softirq handling 2950 * wont try to get same context. 2951 */ 2952 struct tcp_md5sig_pool *tcp_get_md5sig_pool(void) 2953 { 2954 struct tcp_md5sig_pool __percpu *p; 2955 2956 local_bh_disable(); 2957 p = ACCESS_ONCE(tcp_md5sig_pool); 2958 if (p) 2959 return __this_cpu_ptr(p); 2960 2961 local_bh_enable(); 2962 return NULL; 2963 } 2964 EXPORT_SYMBOL(tcp_get_md5sig_pool); 2965 2966 int tcp_md5_hash_header(struct tcp_md5sig_pool *hp, 2967 const struct tcphdr *th) 2968 { 2969 struct scatterlist sg; 2970 struct tcphdr hdr; 2971 int err; 2972 2973 /* We are not allowed to change tcphdr, make a local copy */ 2974 memcpy(&hdr, th, sizeof(hdr)); 2975 hdr.check = 0; 2976 2977 /* options aren't included in the hash */ 2978 sg_init_one(&sg, &hdr, sizeof(hdr)); 2979 err = crypto_hash_update(&hp->md5_desc, &sg, sizeof(hdr)); 2980 return err; 2981 } 2982 EXPORT_SYMBOL(tcp_md5_hash_header); 2983 2984 int tcp_md5_hash_skb_data(struct tcp_md5sig_pool *hp, 2985 const struct sk_buff *skb, unsigned int header_len) 2986 { 2987 struct scatterlist sg; 2988 const struct tcphdr *tp = tcp_hdr(skb); 2989 struct hash_desc *desc = &hp->md5_desc; 2990 unsigned int i; 2991 const unsigned int head_data_len = skb_headlen(skb) > header_len ? 2992 skb_headlen(skb) - header_len : 0; 2993 const struct skb_shared_info *shi = skb_shinfo(skb); 2994 struct sk_buff *frag_iter; 2995 2996 sg_init_table(&sg, 1); 2997 2998 sg_set_buf(&sg, ((u8 *) tp) + header_len, head_data_len); 2999 if (crypto_hash_update(desc, &sg, head_data_len)) 3000 return 1; 3001 3002 for (i = 0; i < shi->nr_frags; ++i) { 3003 const struct skb_frag_struct *f = &shi->frags[i]; 3004 unsigned int offset = f->page_offset; 3005 struct page *page = skb_frag_page(f) + (offset >> PAGE_SHIFT); 3006 3007 sg_set_page(&sg, page, skb_frag_size(f), 3008 offset_in_page(offset)); 3009 if (crypto_hash_update(desc, &sg, skb_frag_size(f))) 3010 return 1; 3011 } 3012 3013 skb_walk_frags(skb, frag_iter) 3014 if (tcp_md5_hash_skb_data(hp, frag_iter, 0)) 3015 return 1; 3016 3017 return 0; 3018 } 3019 EXPORT_SYMBOL(tcp_md5_hash_skb_data); 3020 3021 int tcp_md5_hash_key(struct tcp_md5sig_pool *hp, const struct tcp_md5sig_key *key) 3022 { 3023 struct scatterlist sg; 3024 3025 sg_init_one(&sg, key->key, key->keylen); 3026 return crypto_hash_update(&hp->md5_desc, &sg, key->keylen); 3027 } 3028 EXPORT_SYMBOL(tcp_md5_hash_key); 3029 3030 #endif 3031 3032 void tcp_done(struct sock *sk) 3033 { 3034 struct request_sock *req = tcp_sk(sk)->fastopen_rsk; 3035 3036 if (sk->sk_state == TCP_SYN_SENT || sk->sk_state == TCP_SYN_RECV) 3037 TCP_INC_STATS_BH(sock_net(sk), TCP_MIB_ATTEMPTFAILS); 3038 3039 tcp_set_state(sk, TCP_CLOSE); 3040 tcp_clear_xmit_timers(sk); 3041 if (req != NULL) 3042 reqsk_fastopen_remove(sk, req, false); 3043 3044 sk->sk_shutdown = SHUTDOWN_MASK; 3045 3046 if (!sock_flag(sk, SOCK_DEAD)) 3047 sk->sk_state_change(sk); 3048 else 3049 inet_csk_destroy_sock(sk); 3050 } 3051 EXPORT_SYMBOL_GPL(tcp_done); 3052 3053 extern struct tcp_congestion_ops tcp_reno; 3054 3055 static __initdata unsigned long thash_entries; 3056 static int __init set_thash_entries(char *str) 3057 { 3058 ssize_t ret; 3059 3060 if (!str) 3061 return 0; 3062 3063 ret = kstrtoul(str, 0, &thash_entries); 3064 if (ret) 3065 return 0; 3066 3067 return 1; 3068 } 3069 __setup("thash_entries=", set_thash_entries); 3070 3071 void tcp_init_mem(struct net *net) 3072 { 3073 unsigned long limit = nr_free_buffer_pages() / 8; 3074 limit = max(limit, 128UL); 3075 net->ipv4.sysctl_tcp_mem[0] = limit / 4 * 3; 3076 net->ipv4.sysctl_tcp_mem[1] = limit; 3077 net->ipv4.sysctl_tcp_mem[2] = net->ipv4.sysctl_tcp_mem[0] * 2; 3078 } 3079 3080 void __init tcp_init(void) 3081 { 3082 struct sk_buff *skb = NULL; 3083 unsigned long limit; 3084 int max_rshare, max_wshare, cnt; 3085 unsigned int i; 3086 3087 BUILD_BUG_ON(sizeof(struct tcp_skb_cb) > sizeof(skb->cb)); 3088 3089 percpu_counter_init(&tcp_sockets_allocated, 0); 3090 percpu_counter_init(&tcp_orphan_count, 0); 3091 tcp_hashinfo.bind_bucket_cachep = 3092 kmem_cache_create("tcp_bind_bucket", 3093 sizeof(struct inet_bind_bucket), 0, 3094 SLAB_HWCACHE_ALIGN|SLAB_PANIC, NULL); 3095 3096 /* Size and allocate the main established and bind bucket 3097 * hash tables. 3098 * 3099 * The methodology is similar to that of the buffer cache. 3100 */ 3101 tcp_hashinfo.ehash = 3102 alloc_large_system_hash("TCP established", 3103 sizeof(struct inet_ehash_bucket), 3104 thash_entries, 3105 17, /* one slot per 128 KB of memory */ 3106 0, 3107 NULL, 3108 &tcp_hashinfo.ehash_mask, 3109 0, 3110 thash_entries ? 0 : 512 * 1024); 3111 for (i = 0; i <= tcp_hashinfo.ehash_mask; i++) { 3112 INIT_HLIST_NULLS_HEAD(&tcp_hashinfo.ehash[i].chain, i); 3113 INIT_HLIST_NULLS_HEAD(&tcp_hashinfo.ehash[i].twchain, i); 3114 } 3115 if (inet_ehash_locks_alloc(&tcp_hashinfo)) 3116 panic("TCP: failed to alloc ehash_locks"); 3117 tcp_hashinfo.bhash = 3118 alloc_large_system_hash("TCP bind", 3119 sizeof(struct inet_bind_hashbucket), 3120 tcp_hashinfo.ehash_mask + 1, 3121 17, /* one slot per 128 KB of memory */ 3122 0, 3123 &tcp_hashinfo.bhash_size, 3124 NULL, 3125 0, 3126 64 * 1024); 3127 tcp_hashinfo.bhash_size = 1U << tcp_hashinfo.bhash_size; 3128 for (i = 0; i < tcp_hashinfo.bhash_size; i++) { 3129 spin_lock_init(&tcp_hashinfo.bhash[i].lock); 3130 INIT_HLIST_HEAD(&tcp_hashinfo.bhash[i].chain); 3131 } 3132 3133 3134 cnt = tcp_hashinfo.ehash_mask + 1; 3135 3136 tcp_death_row.sysctl_max_tw_buckets = cnt / 2; 3137 sysctl_tcp_max_orphans = cnt / 2; 3138 sysctl_max_syn_backlog = max(128, cnt / 256); 3139 3140 tcp_init_mem(&init_net); 3141 /* Set per-socket limits to no more than 1/128 the pressure threshold */ 3142 limit = nr_free_buffer_pages() << (PAGE_SHIFT - 7); 3143 max_wshare = min(4UL*1024*1024, limit); 3144 max_rshare = min(6UL*1024*1024, limit); 3145 3146 sysctl_tcp_wmem[0] = SK_MEM_QUANTUM; 3147 sysctl_tcp_wmem[1] = 16*1024; 3148 sysctl_tcp_wmem[2] = max(64*1024, max_wshare); 3149 3150 sysctl_tcp_rmem[0] = SK_MEM_QUANTUM; 3151 sysctl_tcp_rmem[1] = 87380; 3152 sysctl_tcp_rmem[2] = max(87380, max_rshare); 3153 3154 pr_info("Hash tables configured (established %u bind %u)\n", 3155 tcp_hashinfo.ehash_mask + 1, tcp_hashinfo.bhash_size); 3156 3157 tcp_metrics_init(); 3158 3159 tcp_register_congestion_control(&tcp_reno); 3160 3161 tcp_tasklet_init(); 3162 } 3163