1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * INET An implementation of the TCP/IP protocol suite for the LINUX 4 * operating system. INET is implemented using the BSD Socket 5 * interface as the means of communication with the user level. 6 * 7 * Implementation of the Transmission Control Protocol(TCP). 8 * 9 * Authors: Ross Biro 10 * Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG> 11 * Mark Evans, <evansmp@uhura.aston.ac.uk> 12 * Corey Minyard <wf-rch!minyard@relay.EU.net> 13 * Florian La Roche, <flla@stud.uni-sb.de> 14 * Charles Hedrick, <hedrick@klinzhai.rutgers.edu> 15 * Linus Torvalds, <torvalds@cs.helsinki.fi> 16 * Alan Cox, <gw4pts@gw4pts.ampr.org> 17 * Matthew Dillon, <dillon@apollo.west.oic.com> 18 * Arnt Gulbrandsen, <agulbra@nvg.unit.no> 19 * Jorge Cwik, <jorge@laser.satlink.net> 20 * 21 * Fixes: 22 * Alan Cox : Numerous verify_area() calls 23 * Alan Cox : Set the ACK bit on a reset 24 * Alan Cox : Stopped it crashing if it closed while 25 * sk->inuse=1 and was trying to connect 26 * (tcp_err()). 27 * Alan Cox : All icmp error handling was broken 28 * pointers passed where wrong and the 29 * socket was looked up backwards. Nobody 30 * tested any icmp error code obviously. 31 * Alan Cox : tcp_err() now handled properly. It 32 * wakes people on errors. poll 33 * behaves and the icmp error race 34 * has gone by moving it into sock.c 35 * Alan Cox : tcp_send_reset() fixed to work for 36 * everything not just packets for 37 * unknown sockets. 38 * Alan Cox : tcp option processing. 39 * Alan Cox : Reset tweaked (still not 100%) [Had 40 * syn rule wrong] 41 * Herp Rosmanith : More reset fixes 42 * Alan Cox : No longer acks invalid rst frames. 43 * Acking any kind of RST is right out. 44 * Alan Cox : Sets an ignore me flag on an rst 45 * receive otherwise odd bits of prattle 46 * escape still 47 * Alan Cox : Fixed another acking RST frame bug. 48 * Should stop LAN workplace lockups. 49 * Alan Cox : Some tidyups using the new skb list 50 * facilities 51 * Alan Cox : sk->keepopen now seems to work 52 * Alan Cox : Pulls options out correctly on accepts 53 * Alan Cox : Fixed assorted sk->rqueue->next errors 54 * Alan Cox : PSH doesn't end a TCP read. Switched a 55 * bit to skb ops. 56 * Alan Cox : Tidied tcp_data to avoid a potential 57 * nasty. 58 * Alan Cox : Added some better commenting, as the 59 * tcp is hard to follow 60 * Alan Cox : Removed incorrect check for 20 * psh 61 * Michael O'Reilly : ack < copied bug fix. 62 * Johannes Stille : Misc tcp fixes (not all in yet). 63 * Alan Cox : FIN with no memory -> CRASH 64 * Alan Cox : Added socket option proto entries. 65 * Also added awareness of them to accept. 66 * Alan Cox : Added TCP options (SOL_TCP) 67 * Alan Cox : Switched wakeup calls to callbacks, 68 * so the kernel can layer network 69 * sockets. 70 * Alan Cox : Use ip_tos/ip_ttl settings. 71 * Alan Cox : Handle FIN (more) properly (we hope). 72 * Alan Cox : RST frames sent on unsynchronised 73 * state ack error. 74 * Alan Cox : Put in missing check for SYN bit. 75 * Alan Cox : Added tcp_select_window() aka NET2E 76 * window non shrink trick. 77 * Alan Cox : Added a couple of small NET2E timer 78 * fixes 79 * Charles Hedrick : TCP fixes 80 * Toomas Tamm : TCP window fixes 81 * Alan Cox : Small URG fix to rlogin ^C ack fight 82 * Charles Hedrick : Rewrote most of it to actually work 83 * Linus : Rewrote tcp_read() and URG handling 84 * completely 85 * Gerhard Koerting: Fixed some missing timer handling 86 * Matthew Dillon : Reworked TCP machine states as per RFC 87 * Gerhard Koerting: PC/TCP workarounds 88 * Adam Caldwell : Assorted timer/timing errors 89 * Matthew Dillon : Fixed another RST bug 90 * Alan Cox : Move to kernel side addressing changes. 91 * Alan Cox : Beginning work on TCP fastpathing 92 * (not yet usable) 93 * Arnt Gulbrandsen: Turbocharged tcp_check() routine. 94 * Alan Cox : TCP fast path debugging 95 * Alan Cox : Window clamping 96 * Michael Riepe : Bug in tcp_check() 97 * Matt Dillon : More TCP improvements and RST bug fixes 98 * Matt Dillon : Yet more small nasties remove from the 99 * TCP code (Be very nice to this man if 100 * tcp finally works 100%) 8) 101 * Alan Cox : BSD accept semantics. 102 * Alan Cox : Reset on closedown bug. 103 * Peter De Schrijver : ENOTCONN check missing in tcp_sendto(). 104 * Michael Pall : Handle poll() after URG properly in 105 * all cases. 106 * Michael Pall : Undo the last fix in tcp_read_urg() 107 * (multi URG PUSH broke rlogin). 108 * Michael Pall : Fix the multi URG PUSH problem in 109 * tcp_readable(), poll() after URG 110 * works now. 111 * Michael Pall : recv(...,MSG_OOB) never blocks in the 112 * BSD api. 113 * Alan Cox : Changed the semantics of sk->socket to 114 * fix a race and a signal problem with 115 * accept() and async I/O. 116 * Alan Cox : Relaxed the rules on tcp_sendto(). 117 * Yury Shevchuk : Really fixed accept() blocking problem. 118 * Craig I. Hagan : Allow for BSD compatible TIME_WAIT for 119 * clients/servers which listen in on 120 * fixed ports. 121 * Alan Cox : Cleaned the above up and shrank it to 122 * a sensible code size. 123 * Alan Cox : Self connect lockup fix. 124 * Alan Cox : No connect to multicast. 125 * Ross Biro : Close unaccepted children on master 126 * socket close. 127 * Alan Cox : Reset tracing code. 128 * Alan Cox : Spurious resets on shutdown. 129 * Alan Cox : Giant 15 minute/60 second timer error 130 * Alan Cox : Small whoops in polling before an 131 * accept. 132 * Alan Cox : Kept the state trace facility since 133 * it's handy for debugging. 134 * Alan Cox : More reset handler fixes. 135 * Alan Cox : Started rewriting the code based on 136 * the RFC's for other useful protocol 137 * references see: Comer, KA9Q NOS, and 138 * for a reference on the difference 139 * between specifications and how BSD 140 * works see the 4.4lite source. 141 * A.N.Kuznetsov : Don't time wait on completion of tidy 142 * close. 143 * Linus Torvalds : Fin/Shutdown & copied_seq changes. 144 * Linus Torvalds : Fixed BSD port reuse to work first syn 145 * Alan Cox : Reimplemented timers as per the RFC 146 * and using multiple timers for sanity. 147 * Alan Cox : Small bug fixes, and a lot of new 148 * comments. 149 * Alan Cox : Fixed dual reader crash by locking 150 * the buffers (much like datagram.c) 151 * Alan Cox : Fixed stuck sockets in probe. A probe 152 * now gets fed up of retrying without 153 * (even a no space) answer. 154 * Alan Cox : Extracted closing code better 155 * Alan Cox : Fixed the closing state machine to 156 * resemble the RFC. 157 * Alan Cox : More 'per spec' fixes. 158 * Jorge Cwik : Even faster checksumming. 159 * Alan Cox : tcp_data() doesn't ack illegal PSH 160 * only frames. At least one pc tcp stack 161 * generates them. 162 * Alan Cox : Cache last socket. 163 * Alan Cox : Per route irtt. 164 * Matt Day : poll()->select() match BSD precisely on error 165 * Alan Cox : New buffers 166 * Marc Tamsky : Various sk->prot->retransmits and 167 * sk->retransmits misupdating fixed. 168 * Fixed tcp_write_timeout: stuck close, 169 * and TCP syn retries gets used now. 170 * Mark Yarvis : In tcp_read_wakeup(), don't send an 171 * ack if state is TCP_CLOSED. 172 * Alan Cox : Look up device on a retransmit - routes may 173 * change. Doesn't yet cope with MSS shrink right 174 * but it's a start! 175 * Marc Tamsky : Closing in closing fixes. 176 * Mike Shaver : RFC1122 verifications. 177 * Alan Cox : rcv_saddr errors. 178 * Alan Cox : Block double connect(). 179 * Alan Cox : Small hooks for enSKIP. 180 * Alexey Kuznetsov: Path MTU discovery. 181 * Alan Cox : Support soft errors. 182 * Alan Cox : Fix MTU discovery pathological case 183 * when the remote claims no mtu! 184 * Marc Tamsky : TCP_CLOSE fix. 185 * Colin (G3TNE) : Send a reset on syn ack replies in 186 * window but wrong (fixes NT lpd problems) 187 * Pedro Roque : Better TCP window handling, delayed ack. 188 * Joerg Reuter : No modification of locked buffers in 189 * tcp_do_retransmit() 190 * Eric Schenk : Changed receiver side silly window 191 * avoidance algorithm to BSD style 192 * algorithm. This doubles throughput 193 * against machines running Solaris, 194 * and seems to result in general 195 * improvement. 196 * Stefan Magdalinski : adjusted tcp_readable() to fix FIONREAD 197 * Willy Konynenberg : Transparent proxying support. 198 * Mike McLagan : Routing by source 199 * Keith Owens : Do proper merging with partial SKB's in 200 * tcp_do_sendmsg to avoid burstiness. 201 * Eric Schenk : Fix fast close down bug with 202 * shutdown() followed by close(). 203 * Andi Kleen : Make poll agree with SIGIO 204 * Salvatore Sanfilippo : Support SO_LINGER with linger == 1 and 205 * lingertime == 0 (RFC 793 ABORT Call) 206 * Hirokazu Takahashi : Use copy_from_user() instead of 207 * csum_and_copy_from_user() if possible. 208 * 209 * Description of States: 210 * 211 * TCP_SYN_SENT sent a connection request, waiting for ack 212 * 213 * TCP_SYN_RECV received a connection request, sent ack, 214 * waiting for final ack in three-way handshake. 215 * 216 * TCP_ESTABLISHED connection established 217 * 218 * TCP_FIN_WAIT1 our side has shutdown, waiting to complete 219 * transmission of remaining buffered data 220 * 221 * TCP_FIN_WAIT2 all buffered data sent, waiting for remote 222 * to shutdown 223 * 224 * TCP_CLOSING both sides have shutdown but we still have 225 * data we have to finish sending 226 * 227 * TCP_TIME_WAIT timeout to catch resent junk before entering 228 * closed, can only be entered from FIN_WAIT2 229 * or CLOSING. Required because the other end 230 * may not have gotten our last ACK causing it 231 * to retransmit the data packet (which we ignore) 232 * 233 * TCP_CLOSE_WAIT remote side has shutdown and is waiting for 234 * us to finish writing our data and to shutdown 235 * (we have to close() to move on to LAST_ACK) 236 * 237 * TCP_LAST_ACK out side has shutdown after remote has 238 * shutdown. There may still be data in our 239 * buffer that we have to finish sending 240 * 241 * TCP_CLOSE socket is finished 242 */ 243 244 #define pr_fmt(fmt) "TCP: " fmt 245 246 #include <crypto/hash.h> 247 #include <linux/kernel.h> 248 #include <linux/module.h> 249 #include <linux/types.h> 250 #include <linux/fcntl.h> 251 #include <linux/poll.h> 252 #include <linux/inet_diag.h> 253 #include <linux/init.h> 254 #include <linux/fs.h> 255 #include <linux/skbuff.h> 256 #include <linux/scatterlist.h> 257 #include <linux/splice.h> 258 #include <linux/net.h> 259 #include <linux/socket.h> 260 #include <linux/random.h> 261 #include <linux/memblock.h> 262 #include <linux/highmem.h> 263 #include <linux/swap.h> 264 #include <linux/cache.h> 265 #include <linux/err.h> 266 #include <linux/time.h> 267 #include <linux/slab.h> 268 #include <linux/errqueue.h> 269 #include <linux/static_key.h> 270 271 #include <net/icmp.h> 272 #include <net/inet_common.h> 273 #include <net/tcp.h> 274 #include <net/xfrm.h> 275 #include <net/ip.h> 276 #include <net/sock.h> 277 278 #include <linux/uaccess.h> 279 #include <asm/ioctls.h> 280 #include <net/busy_poll.h> 281 282 struct percpu_counter tcp_orphan_count; 283 EXPORT_SYMBOL_GPL(tcp_orphan_count); 284 285 long sysctl_tcp_mem[3] __read_mostly; 286 EXPORT_SYMBOL(sysctl_tcp_mem); 287 288 atomic_long_t tcp_memory_allocated; /* Current allocated memory. */ 289 EXPORT_SYMBOL(tcp_memory_allocated); 290 291 #if IS_ENABLED(CONFIG_SMC) 292 DEFINE_STATIC_KEY_FALSE(tcp_have_smc); 293 EXPORT_SYMBOL(tcp_have_smc); 294 #endif 295 296 /* 297 * Current number of TCP sockets. 298 */ 299 struct percpu_counter tcp_sockets_allocated; 300 EXPORT_SYMBOL(tcp_sockets_allocated); 301 302 /* 303 * TCP splice context 304 */ 305 struct tcp_splice_state { 306 struct pipe_inode_info *pipe; 307 size_t len; 308 unsigned int flags; 309 }; 310 311 /* 312 * Pressure flag: try to collapse. 313 * Technical note: it is used by multiple contexts non atomically. 314 * All the __sk_mem_schedule() is of this nature: accounting 315 * is strict, actions are advisory and have some latency. 316 */ 317 unsigned long tcp_memory_pressure __read_mostly; 318 EXPORT_SYMBOL_GPL(tcp_memory_pressure); 319 320 void tcp_enter_memory_pressure(struct sock *sk) 321 { 322 unsigned long val; 323 324 if (tcp_memory_pressure) 325 return; 326 val = jiffies; 327 328 if (!val) 329 val--; 330 if (!cmpxchg(&tcp_memory_pressure, 0, val)) 331 NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPMEMORYPRESSURES); 332 } 333 EXPORT_SYMBOL_GPL(tcp_enter_memory_pressure); 334 335 void tcp_leave_memory_pressure(struct sock *sk) 336 { 337 unsigned long val; 338 339 if (!tcp_memory_pressure) 340 return; 341 val = xchg(&tcp_memory_pressure, 0); 342 if (val) 343 NET_ADD_STATS(sock_net(sk), LINUX_MIB_TCPMEMORYPRESSURESCHRONO, 344 jiffies_to_msecs(jiffies - val)); 345 } 346 EXPORT_SYMBOL_GPL(tcp_leave_memory_pressure); 347 348 /* Convert seconds to retransmits based on initial and max timeout */ 349 static u8 secs_to_retrans(int seconds, int timeout, int rto_max) 350 { 351 u8 res = 0; 352 353 if (seconds > 0) { 354 int period = timeout; 355 356 res = 1; 357 while (seconds > period && res < 255) { 358 res++; 359 timeout <<= 1; 360 if (timeout > rto_max) 361 timeout = rto_max; 362 period += timeout; 363 } 364 } 365 return res; 366 } 367 368 /* Convert retransmits to seconds based on initial and max timeout */ 369 static int retrans_to_secs(u8 retrans, int timeout, int rto_max) 370 { 371 int period = 0; 372 373 if (retrans > 0) { 374 period = timeout; 375 while (--retrans) { 376 timeout <<= 1; 377 if (timeout > rto_max) 378 timeout = rto_max; 379 period += timeout; 380 } 381 } 382 return period; 383 } 384 385 static u64 tcp_compute_delivery_rate(const struct tcp_sock *tp) 386 { 387 u32 rate = READ_ONCE(tp->rate_delivered); 388 u32 intv = READ_ONCE(tp->rate_interval_us); 389 u64 rate64 = 0; 390 391 if (rate && intv) { 392 rate64 = (u64)rate * tp->mss_cache * USEC_PER_SEC; 393 do_div(rate64, intv); 394 } 395 return rate64; 396 } 397 398 /* Address-family independent initialization for a tcp_sock. 399 * 400 * NOTE: A lot of things set to zero explicitly by call to 401 * sk_alloc() so need not be done here. 402 */ 403 void tcp_init_sock(struct sock *sk) 404 { 405 struct inet_connection_sock *icsk = inet_csk(sk); 406 struct tcp_sock *tp = tcp_sk(sk); 407 408 tp->out_of_order_queue = RB_ROOT; 409 sk->tcp_rtx_queue = RB_ROOT; 410 tcp_init_xmit_timers(sk); 411 INIT_LIST_HEAD(&tp->tsq_node); 412 INIT_LIST_HEAD(&tp->tsorted_sent_queue); 413 414 icsk->icsk_rto = TCP_TIMEOUT_INIT; 415 tp->mdev_us = jiffies_to_usecs(TCP_TIMEOUT_INIT); 416 minmax_reset(&tp->rtt_min, tcp_jiffies32, ~0U); 417 418 /* So many TCP implementations out there (incorrectly) count the 419 * initial SYN frame in their delayed-ACK and congestion control 420 * algorithms that we must have the following bandaid to talk 421 * efficiently to them. -DaveM 422 */ 423 tp->snd_cwnd = TCP_INIT_CWND; 424 425 /* There's a bubble in the pipe until at least the first ACK. */ 426 tp->app_limited = ~0U; 427 428 /* See draft-stevens-tcpca-spec-01 for discussion of the 429 * initialization of these values. 430 */ 431 tp->snd_ssthresh = TCP_INFINITE_SSTHRESH; 432 tp->snd_cwnd_clamp = ~0; 433 tp->mss_cache = TCP_MSS_DEFAULT; 434 435 tp->reordering = sock_net(sk)->ipv4.sysctl_tcp_reordering; 436 tcp_assign_congestion_control(sk); 437 438 tp->tsoffset = 0; 439 tp->rack.reo_wnd_steps = 1; 440 441 sk->sk_state = TCP_CLOSE; 442 443 sk->sk_write_space = sk_stream_write_space; 444 sock_set_flag(sk, SOCK_USE_WRITE_QUEUE); 445 446 icsk->icsk_sync_mss = tcp_sync_mss; 447 448 sk->sk_sndbuf = sock_net(sk)->ipv4.sysctl_tcp_wmem[1]; 449 sk->sk_rcvbuf = sock_net(sk)->ipv4.sysctl_tcp_rmem[1]; 450 451 sk_sockets_allocated_inc(sk); 452 sk->sk_route_forced_caps = NETIF_F_GSO; 453 } 454 EXPORT_SYMBOL(tcp_init_sock); 455 456 static void tcp_tx_timestamp(struct sock *sk, u16 tsflags) 457 { 458 struct sk_buff *skb = tcp_write_queue_tail(sk); 459 460 if (tsflags && skb) { 461 struct skb_shared_info *shinfo = skb_shinfo(skb); 462 struct tcp_skb_cb *tcb = TCP_SKB_CB(skb); 463 464 sock_tx_timestamp(sk, tsflags, &shinfo->tx_flags); 465 if (tsflags & SOF_TIMESTAMPING_TX_ACK) 466 tcb->txstamp_ack = 1; 467 if (tsflags & SOF_TIMESTAMPING_TX_RECORD_MASK) 468 shinfo->tskey = TCP_SKB_CB(skb)->seq + skb->len - 1; 469 } 470 } 471 472 static inline bool tcp_stream_is_readable(const struct tcp_sock *tp, 473 int target, struct sock *sk) 474 { 475 return (tp->rcv_nxt - tp->copied_seq >= target) || 476 (sk->sk_prot->stream_memory_read ? 477 sk->sk_prot->stream_memory_read(sk) : false); 478 } 479 480 /* 481 * Wait for a TCP event. 482 * 483 * Note that we don't need to lock the socket, as the upper poll layers 484 * take care of normal races (between the test and the event) and we don't 485 * go look at any of the socket buffers directly. 486 */ 487 __poll_t tcp_poll(struct file *file, struct socket *sock, poll_table *wait) 488 { 489 __poll_t mask; 490 struct sock *sk = sock->sk; 491 const struct tcp_sock *tp = tcp_sk(sk); 492 int state; 493 494 sock_poll_wait(file, sock, wait); 495 496 state = inet_sk_state_load(sk); 497 if (state == TCP_LISTEN) 498 return inet_csk_listen_poll(sk); 499 500 /* Socket is not locked. We are protected from async events 501 * by poll logic and correct handling of state changes 502 * made by other threads is impossible in any case. 503 */ 504 505 mask = 0; 506 507 /* 508 * EPOLLHUP is certainly not done right. But poll() doesn't 509 * have a notion of HUP in just one direction, and for a 510 * socket the read side is more interesting. 511 * 512 * Some poll() documentation says that EPOLLHUP is incompatible 513 * with the EPOLLOUT/POLLWR flags, so somebody should check this 514 * all. But careful, it tends to be safer to return too many 515 * bits than too few, and you can easily break real applications 516 * if you don't tell them that something has hung up! 517 * 518 * Check-me. 519 * 520 * Check number 1. EPOLLHUP is _UNMASKABLE_ event (see UNIX98 and 521 * our fs/select.c). It means that after we received EOF, 522 * poll always returns immediately, making impossible poll() on write() 523 * in state CLOSE_WAIT. One solution is evident --- to set EPOLLHUP 524 * if and only if shutdown has been made in both directions. 525 * Actually, it is interesting to look how Solaris and DUX 526 * solve this dilemma. I would prefer, if EPOLLHUP were maskable, 527 * then we could set it on SND_SHUTDOWN. BTW examples given 528 * in Stevens' books assume exactly this behaviour, it explains 529 * why EPOLLHUP is incompatible with EPOLLOUT. --ANK 530 * 531 * NOTE. Check for TCP_CLOSE is added. The goal is to prevent 532 * blocking on fresh not-connected or disconnected socket. --ANK 533 */ 534 if (sk->sk_shutdown == SHUTDOWN_MASK || state == TCP_CLOSE) 535 mask |= EPOLLHUP; 536 if (sk->sk_shutdown & RCV_SHUTDOWN) 537 mask |= EPOLLIN | EPOLLRDNORM | EPOLLRDHUP; 538 539 /* Connected or passive Fast Open socket? */ 540 if (state != TCP_SYN_SENT && 541 (state != TCP_SYN_RECV || tp->fastopen_rsk)) { 542 int target = sock_rcvlowat(sk, 0, INT_MAX); 543 544 if (tp->urg_seq == tp->copied_seq && 545 !sock_flag(sk, SOCK_URGINLINE) && 546 tp->urg_data) 547 target++; 548 549 if (tcp_stream_is_readable(tp, target, sk)) 550 mask |= EPOLLIN | EPOLLRDNORM; 551 552 if (!(sk->sk_shutdown & SEND_SHUTDOWN)) { 553 if (sk_stream_is_writeable(sk)) { 554 mask |= EPOLLOUT | EPOLLWRNORM; 555 } else { /* send SIGIO later */ 556 sk_set_bit(SOCKWQ_ASYNC_NOSPACE, sk); 557 set_bit(SOCK_NOSPACE, &sk->sk_socket->flags); 558 559 /* Race breaker. If space is freed after 560 * wspace test but before the flags are set, 561 * IO signal will be lost. Memory barrier 562 * pairs with the input side. 563 */ 564 smp_mb__after_atomic(); 565 if (sk_stream_is_writeable(sk)) 566 mask |= EPOLLOUT | EPOLLWRNORM; 567 } 568 } else 569 mask |= EPOLLOUT | EPOLLWRNORM; 570 571 if (tp->urg_data & TCP_URG_VALID) 572 mask |= EPOLLPRI; 573 } else if (state == TCP_SYN_SENT && inet_sk(sk)->defer_connect) { 574 /* Active TCP fastopen socket with defer_connect 575 * Return EPOLLOUT so application can call write() 576 * in order for kernel to generate SYN+data 577 */ 578 mask |= EPOLLOUT | EPOLLWRNORM; 579 } 580 /* This barrier is coupled with smp_wmb() in tcp_reset() */ 581 smp_rmb(); 582 if (sk->sk_err || !skb_queue_empty(&sk->sk_error_queue)) 583 mask |= EPOLLERR; 584 585 return mask; 586 } 587 EXPORT_SYMBOL(tcp_poll); 588 589 int tcp_ioctl(struct sock *sk, int cmd, unsigned long arg) 590 { 591 struct tcp_sock *tp = tcp_sk(sk); 592 int answ; 593 bool slow; 594 595 switch (cmd) { 596 case SIOCINQ: 597 if (sk->sk_state == TCP_LISTEN) 598 return -EINVAL; 599 600 slow = lock_sock_fast(sk); 601 answ = tcp_inq(sk); 602 unlock_sock_fast(sk, slow); 603 break; 604 case SIOCATMARK: 605 answ = tp->urg_data && tp->urg_seq == tp->copied_seq; 606 break; 607 case SIOCOUTQ: 608 if (sk->sk_state == TCP_LISTEN) 609 return -EINVAL; 610 611 if ((1 << sk->sk_state) & (TCPF_SYN_SENT | TCPF_SYN_RECV)) 612 answ = 0; 613 else 614 answ = tp->write_seq - tp->snd_una; 615 break; 616 case SIOCOUTQNSD: 617 if (sk->sk_state == TCP_LISTEN) 618 return -EINVAL; 619 620 if ((1 << sk->sk_state) & (TCPF_SYN_SENT | TCPF_SYN_RECV)) 621 answ = 0; 622 else 623 answ = tp->write_seq - tp->snd_nxt; 624 break; 625 default: 626 return -ENOIOCTLCMD; 627 } 628 629 return put_user(answ, (int __user *)arg); 630 } 631 EXPORT_SYMBOL(tcp_ioctl); 632 633 static inline void tcp_mark_push(struct tcp_sock *tp, struct sk_buff *skb) 634 { 635 TCP_SKB_CB(skb)->tcp_flags |= TCPHDR_PSH; 636 tp->pushed_seq = tp->write_seq; 637 } 638 639 static inline bool forced_push(const struct tcp_sock *tp) 640 { 641 return after(tp->write_seq, tp->pushed_seq + (tp->max_window >> 1)); 642 } 643 644 static void skb_entail(struct sock *sk, struct sk_buff *skb) 645 { 646 struct tcp_sock *tp = tcp_sk(sk); 647 struct tcp_skb_cb *tcb = TCP_SKB_CB(skb); 648 649 skb->csum = 0; 650 tcb->seq = tcb->end_seq = tp->write_seq; 651 tcb->tcp_flags = TCPHDR_ACK; 652 tcb->sacked = 0; 653 __skb_header_release(skb); 654 tcp_add_write_queue_tail(sk, skb); 655 sk->sk_wmem_queued += skb->truesize; 656 sk_mem_charge(sk, skb->truesize); 657 if (tp->nonagle & TCP_NAGLE_PUSH) 658 tp->nonagle &= ~TCP_NAGLE_PUSH; 659 660 tcp_slow_start_after_idle_check(sk); 661 } 662 663 static inline void tcp_mark_urg(struct tcp_sock *tp, int flags) 664 { 665 if (flags & MSG_OOB) 666 tp->snd_up = tp->write_seq; 667 } 668 669 /* If a not yet filled skb is pushed, do not send it if 670 * we have data packets in Qdisc or NIC queues : 671 * Because TX completion will happen shortly, it gives a chance 672 * to coalesce future sendmsg() payload into this skb, without 673 * need for a timer, and with no latency trade off. 674 * As packets containing data payload have a bigger truesize 675 * than pure acks (dataless) packets, the last checks prevent 676 * autocorking if we only have an ACK in Qdisc/NIC queues, 677 * or if TX completion was delayed after we processed ACK packet. 678 */ 679 static bool tcp_should_autocork(struct sock *sk, struct sk_buff *skb, 680 int size_goal) 681 { 682 return skb->len < size_goal && 683 sock_net(sk)->ipv4.sysctl_tcp_autocorking && 684 !tcp_rtx_queue_empty(sk) && 685 refcount_read(&sk->sk_wmem_alloc) > skb->truesize; 686 } 687 688 static void tcp_push(struct sock *sk, int flags, int mss_now, 689 int nonagle, int size_goal) 690 { 691 struct tcp_sock *tp = tcp_sk(sk); 692 struct sk_buff *skb; 693 694 skb = tcp_write_queue_tail(sk); 695 if (!skb) 696 return; 697 if (!(flags & MSG_MORE) || forced_push(tp)) 698 tcp_mark_push(tp, skb); 699 700 tcp_mark_urg(tp, flags); 701 702 if (tcp_should_autocork(sk, skb, size_goal)) { 703 704 /* avoid atomic op if TSQ_THROTTLED bit is already set */ 705 if (!test_bit(TSQ_THROTTLED, &sk->sk_tsq_flags)) { 706 NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPAUTOCORKING); 707 set_bit(TSQ_THROTTLED, &sk->sk_tsq_flags); 708 } 709 /* It is possible TX completion already happened 710 * before we set TSQ_THROTTLED. 711 */ 712 if (refcount_read(&sk->sk_wmem_alloc) > skb->truesize) 713 return; 714 } 715 716 if (flags & MSG_MORE) 717 nonagle = TCP_NAGLE_CORK; 718 719 __tcp_push_pending_frames(sk, mss_now, nonagle); 720 } 721 722 static int tcp_splice_data_recv(read_descriptor_t *rd_desc, struct sk_buff *skb, 723 unsigned int offset, size_t len) 724 { 725 struct tcp_splice_state *tss = rd_desc->arg.data; 726 int ret; 727 728 ret = skb_splice_bits(skb, skb->sk, offset, tss->pipe, 729 min(rd_desc->count, len), tss->flags); 730 if (ret > 0) 731 rd_desc->count -= ret; 732 return ret; 733 } 734 735 static int __tcp_splice_read(struct sock *sk, struct tcp_splice_state *tss) 736 { 737 /* Store TCP splice context information in read_descriptor_t. */ 738 read_descriptor_t rd_desc = { 739 .arg.data = tss, 740 .count = tss->len, 741 }; 742 743 return tcp_read_sock(sk, &rd_desc, tcp_splice_data_recv); 744 } 745 746 /** 747 * tcp_splice_read - splice data from TCP socket to a pipe 748 * @sock: socket to splice from 749 * @ppos: position (not valid) 750 * @pipe: pipe to splice to 751 * @len: number of bytes to splice 752 * @flags: splice modifier flags 753 * 754 * Description: 755 * Will read pages from given socket and fill them into a pipe. 756 * 757 **/ 758 ssize_t tcp_splice_read(struct socket *sock, loff_t *ppos, 759 struct pipe_inode_info *pipe, size_t len, 760 unsigned int flags) 761 { 762 struct sock *sk = sock->sk; 763 struct tcp_splice_state tss = { 764 .pipe = pipe, 765 .len = len, 766 .flags = flags, 767 }; 768 long timeo; 769 ssize_t spliced; 770 int ret; 771 772 sock_rps_record_flow(sk); 773 /* 774 * We can't seek on a socket input 775 */ 776 if (unlikely(*ppos)) 777 return -ESPIPE; 778 779 ret = spliced = 0; 780 781 lock_sock(sk); 782 783 timeo = sock_rcvtimeo(sk, sock->file->f_flags & O_NONBLOCK); 784 while (tss.len) { 785 ret = __tcp_splice_read(sk, &tss); 786 if (ret < 0) 787 break; 788 else if (!ret) { 789 if (spliced) 790 break; 791 if (sock_flag(sk, SOCK_DONE)) 792 break; 793 if (sk->sk_err) { 794 ret = sock_error(sk); 795 break; 796 } 797 if (sk->sk_shutdown & RCV_SHUTDOWN) 798 break; 799 if (sk->sk_state == TCP_CLOSE) { 800 /* 801 * This occurs when user tries to read 802 * from never connected socket. 803 */ 804 ret = -ENOTCONN; 805 break; 806 } 807 if (!timeo) { 808 ret = -EAGAIN; 809 break; 810 } 811 /* if __tcp_splice_read() got nothing while we have 812 * an skb in receive queue, we do not want to loop. 813 * This might happen with URG data. 814 */ 815 if (!skb_queue_empty(&sk->sk_receive_queue)) 816 break; 817 sk_wait_data(sk, &timeo, NULL); 818 if (signal_pending(current)) { 819 ret = sock_intr_errno(timeo); 820 break; 821 } 822 continue; 823 } 824 tss.len -= ret; 825 spliced += ret; 826 827 if (!timeo) 828 break; 829 release_sock(sk); 830 lock_sock(sk); 831 832 if (sk->sk_err || sk->sk_state == TCP_CLOSE || 833 (sk->sk_shutdown & RCV_SHUTDOWN) || 834 signal_pending(current)) 835 break; 836 } 837 838 release_sock(sk); 839 840 if (spliced) 841 return spliced; 842 843 return ret; 844 } 845 EXPORT_SYMBOL(tcp_splice_read); 846 847 struct sk_buff *sk_stream_alloc_skb(struct sock *sk, int size, gfp_t gfp, 848 bool force_schedule) 849 { 850 struct sk_buff *skb; 851 852 if (likely(!size)) { 853 skb = sk->sk_tx_skb_cache; 854 if (skb) { 855 skb->truesize = SKB_TRUESIZE(skb_end_offset(skb)); 856 sk->sk_tx_skb_cache = NULL; 857 pskb_trim(skb, 0); 858 INIT_LIST_HEAD(&skb->tcp_tsorted_anchor); 859 skb_shinfo(skb)->tx_flags = 0; 860 memset(TCP_SKB_CB(skb), 0, sizeof(struct tcp_skb_cb)); 861 return skb; 862 } 863 } 864 /* The TCP header must be at least 32-bit aligned. */ 865 size = ALIGN(size, 4); 866 867 if (unlikely(tcp_under_memory_pressure(sk))) 868 sk_mem_reclaim_partial(sk); 869 870 skb = alloc_skb_fclone(size + sk->sk_prot->max_header, gfp); 871 if (likely(skb)) { 872 bool mem_scheduled; 873 874 if (force_schedule) { 875 mem_scheduled = true; 876 sk_forced_mem_schedule(sk, skb->truesize); 877 } else { 878 mem_scheduled = sk_wmem_schedule(sk, skb->truesize); 879 } 880 if (likely(mem_scheduled)) { 881 skb_reserve(skb, sk->sk_prot->max_header); 882 /* 883 * Make sure that we have exactly size bytes 884 * available to the caller, no more, no less. 885 */ 886 skb->reserved_tailroom = skb->end - skb->tail - size; 887 INIT_LIST_HEAD(&skb->tcp_tsorted_anchor); 888 return skb; 889 } 890 __kfree_skb(skb); 891 } else { 892 sk->sk_prot->enter_memory_pressure(sk); 893 sk_stream_moderate_sndbuf(sk); 894 } 895 return NULL; 896 } 897 898 static unsigned int tcp_xmit_size_goal(struct sock *sk, u32 mss_now, 899 int large_allowed) 900 { 901 struct tcp_sock *tp = tcp_sk(sk); 902 u32 new_size_goal, size_goal; 903 904 if (!large_allowed) 905 return mss_now; 906 907 /* Note : tcp_tso_autosize() will eventually split this later */ 908 new_size_goal = sk->sk_gso_max_size - 1 - MAX_TCP_HEADER; 909 new_size_goal = tcp_bound_to_half_wnd(tp, new_size_goal); 910 911 /* We try hard to avoid divides here */ 912 size_goal = tp->gso_segs * mss_now; 913 if (unlikely(new_size_goal < size_goal || 914 new_size_goal >= size_goal + mss_now)) { 915 tp->gso_segs = min_t(u16, new_size_goal / mss_now, 916 sk->sk_gso_max_segs); 917 size_goal = tp->gso_segs * mss_now; 918 } 919 920 return max(size_goal, mss_now); 921 } 922 923 static int tcp_send_mss(struct sock *sk, int *size_goal, int flags) 924 { 925 int mss_now; 926 927 mss_now = tcp_current_mss(sk); 928 *size_goal = tcp_xmit_size_goal(sk, mss_now, !(flags & MSG_OOB)); 929 930 return mss_now; 931 } 932 933 ssize_t do_tcp_sendpages(struct sock *sk, struct page *page, int offset, 934 size_t size, int flags) 935 { 936 struct tcp_sock *tp = tcp_sk(sk); 937 int mss_now, size_goal; 938 int err; 939 ssize_t copied; 940 long timeo = sock_sndtimeo(sk, flags & MSG_DONTWAIT); 941 942 if (IS_ENABLED(CONFIG_DEBUG_VM) && 943 WARN_ONCE(PageSlab(page), "page must not be a Slab one")) 944 return -EINVAL; 945 946 /* Wait for a connection to finish. One exception is TCP Fast Open 947 * (passive side) where data is allowed to be sent before a connection 948 * is fully established. 949 */ 950 if (((1 << sk->sk_state) & ~(TCPF_ESTABLISHED | TCPF_CLOSE_WAIT)) && 951 !tcp_passive_fastopen(sk)) { 952 err = sk_stream_wait_connect(sk, &timeo); 953 if (err != 0) 954 goto out_err; 955 } 956 957 sk_clear_bit(SOCKWQ_ASYNC_NOSPACE, sk); 958 959 mss_now = tcp_send_mss(sk, &size_goal, flags); 960 copied = 0; 961 962 err = -EPIPE; 963 if (sk->sk_err || (sk->sk_shutdown & SEND_SHUTDOWN)) 964 goto out_err; 965 966 while (size > 0) { 967 struct sk_buff *skb = tcp_write_queue_tail(sk); 968 int copy, i; 969 bool can_coalesce; 970 971 if (!skb || (copy = size_goal - skb->len) <= 0 || 972 !tcp_skb_can_collapse_to(skb)) { 973 new_segment: 974 if (!sk_stream_memory_free(sk)) 975 goto wait_for_sndbuf; 976 977 skb = sk_stream_alloc_skb(sk, 0, sk->sk_allocation, 978 tcp_rtx_and_write_queues_empty(sk)); 979 if (!skb) 980 goto wait_for_memory; 981 982 skb_entail(sk, skb); 983 copy = size_goal; 984 } 985 986 if (copy > size) 987 copy = size; 988 989 i = skb_shinfo(skb)->nr_frags; 990 can_coalesce = skb_can_coalesce(skb, i, page, offset); 991 if (!can_coalesce && i >= sysctl_max_skb_frags) { 992 tcp_mark_push(tp, skb); 993 goto new_segment; 994 } 995 if (!sk_wmem_schedule(sk, copy)) 996 goto wait_for_memory; 997 998 if (can_coalesce) { 999 skb_frag_size_add(&skb_shinfo(skb)->frags[i - 1], copy); 1000 } else { 1001 get_page(page); 1002 skb_fill_page_desc(skb, i, page, offset, copy); 1003 } 1004 1005 if (!(flags & MSG_NO_SHARED_FRAGS)) 1006 skb_shinfo(skb)->tx_flags |= SKBTX_SHARED_FRAG; 1007 1008 skb->len += copy; 1009 skb->data_len += copy; 1010 skb->truesize += copy; 1011 sk->sk_wmem_queued += copy; 1012 sk_mem_charge(sk, copy); 1013 skb->ip_summed = CHECKSUM_PARTIAL; 1014 tp->write_seq += copy; 1015 TCP_SKB_CB(skb)->end_seq += copy; 1016 tcp_skb_pcount_set(skb, 0); 1017 1018 if (!copied) 1019 TCP_SKB_CB(skb)->tcp_flags &= ~TCPHDR_PSH; 1020 1021 copied += copy; 1022 offset += copy; 1023 size -= copy; 1024 if (!size) 1025 goto out; 1026 1027 if (skb->len < size_goal || (flags & MSG_OOB)) 1028 continue; 1029 1030 if (forced_push(tp)) { 1031 tcp_mark_push(tp, skb); 1032 __tcp_push_pending_frames(sk, mss_now, TCP_NAGLE_PUSH); 1033 } else if (skb == tcp_send_head(sk)) 1034 tcp_push_one(sk, mss_now); 1035 continue; 1036 1037 wait_for_sndbuf: 1038 set_bit(SOCK_NOSPACE, &sk->sk_socket->flags); 1039 wait_for_memory: 1040 tcp_push(sk, flags & ~MSG_MORE, mss_now, 1041 TCP_NAGLE_PUSH, size_goal); 1042 1043 err = sk_stream_wait_memory(sk, &timeo); 1044 if (err != 0) 1045 goto do_error; 1046 1047 mss_now = tcp_send_mss(sk, &size_goal, flags); 1048 } 1049 1050 out: 1051 if (copied) { 1052 tcp_tx_timestamp(sk, sk->sk_tsflags); 1053 if (!(flags & MSG_SENDPAGE_NOTLAST)) 1054 tcp_push(sk, flags, mss_now, tp->nonagle, size_goal); 1055 } 1056 return copied; 1057 1058 do_error: 1059 if (copied) 1060 goto out; 1061 out_err: 1062 /* make sure we wake any epoll edge trigger waiter */ 1063 if (unlikely(skb_queue_len(&sk->sk_write_queue) == 0 && 1064 err == -EAGAIN)) { 1065 sk->sk_write_space(sk); 1066 tcp_chrono_stop(sk, TCP_CHRONO_SNDBUF_LIMITED); 1067 } 1068 return sk_stream_error(sk, flags, err); 1069 } 1070 EXPORT_SYMBOL_GPL(do_tcp_sendpages); 1071 1072 int tcp_sendpage_locked(struct sock *sk, struct page *page, int offset, 1073 size_t size, int flags) 1074 { 1075 if (!(sk->sk_route_caps & NETIF_F_SG)) 1076 return sock_no_sendpage_locked(sk, page, offset, size, flags); 1077 1078 tcp_rate_check_app_limited(sk); /* is sending application-limited? */ 1079 1080 return do_tcp_sendpages(sk, page, offset, size, flags); 1081 } 1082 EXPORT_SYMBOL_GPL(tcp_sendpage_locked); 1083 1084 int tcp_sendpage(struct sock *sk, struct page *page, int offset, 1085 size_t size, int flags) 1086 { 1087 int ret; 1088 1089 lock_sock(sk); 1090 ret = tcp_sendpage_locked(sk, page, offset, size, flags); 1091 release_sock(sk); 1092 1093 return ret; 1094 } 1095 EXPORT_SYMBOL(tcp_sendpage); 1096 1097 void tcp_free_fastopen_req(struct tcp_sock *tp) 1098 { 1099 if (tp->fastopen_req) { 1100 kfree(tp->fastopen_req); 1101 tp->fastopen_req = NULL; 1102 } 1103 } 1104 1105 static int tcp_sendmsg_fastopen(struct sock *sk, struct msghdr *msg, 1106 int *copied, size_t size, 1107 struct ubuf_info *uarg) 1108 { 1109 struct tcp_sock *tp = tcp_sk(sk); 1110 struct inet_sock *inet = inet_sk(sk); 1111 struct sockaddr *uaddr = msg->msg_name; 1112 int err, flags; 1113 1114 if (!(sock_net(sk)->ipv4.sysctl_tcp_fastopen & TFO_CLIENT_ENABLE) || 1115 (uaddr && msg->msg_namelen >= sizeof(uaddr->sa_family) && 1116 uaddr->sa_family == AF_UNSPEC)) 1117 return -EOPNOTSUPP; 1118 if (tp->fastopen_req) 1119 return -EALREADY; /* Another Fast Open is in progress */ 1120 1121 tp->fastopen_req = kzalloc(sizeof(struct tcp_fastopen_request), 1122 sk->sk_allocation); 1123 if (unlikely(!tp->fastopen_req)) 1124 return -ENOBUFS; 1125 tp->fastopen_req->data = msg; 1126 tp->fastopen_req->size = size; 1127 tp->fastopen_req->uarg = uarg; 1128 1129 if (inet->defer_connect) { 1130 err = tcp_connect(sk); 1131 /* Same failure procedure as in tcp_v4/6_connect */ 1132 if (err) { 1133 tcp_set_state(sk, TCP_CLOSE); 1134 inet->inet_dport = 0; 1135 sk->sk_route_caps = 0; 1136 } 1137 } 1138 flags = (msg->msg_flags & MSG_DONTWAIT) ? O_NONBLOCK : 0; 1139 err = __inet_stream_connect(sk->sk_socket, uaddr, 1140 msg->msg_namelen, flags, 1); 1141 /* fastopen_req could already be freed in __inet_stream_connect 1142 * if the connection times out or gets rst 1143 */ 1144 if (tp->fastopen_req) { 1145 *copied = tp->fastopen_req->copied; 1146 tcp_free_fastopen_req(tp); 1147 inet->defer_connect = 0; 1148 } 1149 return err; 1150 } 1151 1152 int tcp_sendmsg_locked(struct sock *sk, struct msghdr *msg, size_t size) 1153 { 1154 struct tcp_sock *tp = tcp_sk(sk); 1155 struct ubuf_info *uarg = NULL; 1156 struct sk_buff *skb; 1157 struct sockcm_cookie sockc; 1158 int flags, err, copied = 0; 1159 int mss_now = 0, size_goal, copied_syn = 0; 1160 bool process_backlog = false; 1161 bool zc = false; 1162 long timeo; 1163 1164 flags = msg->msg_flags; 1165 1166 if (flags & MSG_ZEROCOPY && size && sock_flag(sk, SOCK_ZEROCOPY)) { 1167 skb = tcp_write_queue_tail(sk); 1168 uarg = sock_zerocopy_realloc(sk, size, skb_zcopy(skb)); 1169 if (!uarg) { 1170 err = -ENOBUFS; 1171 goto out_err; 1172 } 1173 1174 zc = sk->sk_route_caps & NETIF_F_SG; 1175 if (!zc) 1176 uarg->zerocopy = 0; 1177 } 1178 1179 if (unlikely(flags & MSG_FASTOPEN || inet_sk(sk)->defer_connect) && 1180 !tp->repair) { 1181 err = tcp_sendmsg_fastopen(sk, msg, &copied_syn, size, uarg); 1182 if (err == -EINPROGRESS && copied_syn > 0) 1183 goto out; 1184 else if (err) 1185 goto out_err; 1186 } 1187 1188 timeo = sock_sndtimeo(sk, flags & MSG_DONTWAIT); 1189 1190 tcp_rate_check_app_limited(sk); /* is sending application-limited? */ 1191 1192 /* Wait for a connection to finish. One exception is TCP Fast Open 1193 * (passive side) where data is allowed to be sent before a connection 1194 * is fully established. 1195 */ 1196 if (((1 << sk->sk_state) & ~(TCPF_ESTABLISHED | TCPF_CLOSE_WAIT)) && 1197 !tcp_passive_fastopen(sk)) { 1198 err = sk_stream_wait_connect(sk, &timeo); 1199 if (err != 0) 1200 goto do_error; 1201 } 1202 1203 if (unlikely(tp->repair)) { 1204 if (tp->repair_queue == TCP_RECV_QUEUE) { 1205 copied = tcp_send_rcvq(sk, msg, size); 1206 goto out_nopush; 1207 } 1208 1209 err = -EINVAL; 1210 if (tp->repair_queue == TCP_NO_QUEUE) 1211 goto out_err; 1212 1213 /* 'common' sending to sendq */ 1214 } 1215 1216 sockcm_init(&sockc, sk); 1217 if (msg->msg_controllen) { 1218 err = sock_cmsg_send(sk, msg, &sockc); 1219 if (unlikely(err)) { 1220 err = -EINVAL; 1221 goto out_err; 1222 } 1223 } 1224 1225 /* This should be in poll */ 1226 sk_clear_bit(SOCKWQ_ASYNC_NOSPACE, sk); 1227 1228 /* Ok commence sending. */ 1229 copied = 0; 1230 1231 restart: 1232 mss_now = tcp_send_mss(sk, &size_goal, flags); 1233 1234 err = -EPIPE; 1235 if (sk->sk_err || (sk->sk_shutdown & SEND_SHUTDOWN)) 1236 goto do_error; 1237 1238 while (msg_data_left(msg)) { 1239 int copy = 0; 1240 1241 skb = tcp_write_queue_tail(sk); 1242 if (skb) 1243 copy = size_goal - skb->len; 1244 1245 if (copy <= 0 || !tcp_skb_can_collapse_to(skb)) { 1246 bool first_skb; 1247 1248 new_segment: 1249 if (!sk_stream_memory_free(sk)) 1250 goto wait_for_sndbuf; 1251 1252 if (process_backlog && sk_flush_backlog(sk)) { 1253 process_backlog = false; 1254 goto restart; 1255 } 1256 first_skb = tcp_rtx_and_write_queues_empty(sk); 1257 skb = sk_stream_alloc_skb(sk, 0, sk->sk_allocation, 1258 first_skb); 1259 if (!skb) 1260 goto wait_for_memory; 1261 1262 process_backlog = true; 1263 skb->ip_summed = CHECKSUM_PARTIAL; 1264 1265 skb_entail(sk, skb); 1266 copy = size_goal; 1267 1268 /* All packets are restored as if they have 1269 * already been sent. skb_mstamp_ns isn't set to 1270 * avoid wrong rtt estimation. 1271 */ 1272 if (tp->repair) 1273 TCP_SKB_CB(skb)->sacked |= TCPCB_REPAIRED; 1274 } 1275 1276 /* Try to append data to the end of skb. */ 1277 if (copy > msg_data_left(msg)) 1278 copy = msg_data_left(msg); 1279 1280 /* Where to copy to? */ 1281 if (skb_availroom(skb) > 0 && !zc) { 1282 /* We have some space in skb head. Superb! */ 1283 copy = min_t(int, copy, skb_availroom(skb)); 1284 err = skb_add_data_nocache(sk, skb, &msg->msg_iter, copy); 1285 if (err) 1286 goto do_fault; 1287 } else if (!zc) { 1288 bool merge = true; 1289 int i = skb_shinfo(skb)->nr_frags; 1290 struct page_frag *pfrag = sk_page_frag(sk); 1291 1292 if (!sk_page_frag_refill(sk, pfrag)) 1293 goto wait_for_memory; 1294 1295 if (!skb_can_coalesce(skb, i, pfrag->page, 1296 pfrag->offset)) { 1297 if (i >= sysctl_max_skb_frags) { 1298 tcp_mark_push(tp, skb); 1299 goto new_segment; 1300 } 1301 merge = false; 1302 } 1303 1304 copy = min_t(int, copy, pfrag->size - pfrag->offset); 1305 1306 if (!sk_wmem_schedule(sk, copy)) 1307 goto wait_for_memory; 1308 1309 err = skb_copy_to_page_nocache(sk, &msg->msg_iter, skb, 1310 pfrag->page, 1311 pfrag->offset, 1312 copy); 1313 if (err) 1314 goto do_error; 1315 1316 /* Update the skb. */ 1317 if (merge) { 1318 skb_frag_size_add(&skb_shinfo(skb)->frags[i - 1], copy); 1319 } else { 1320 skb_fill_page_desc(skb, i, pfrag->page, 1321 pfrag->offset, copy); 1322 page_ref_inc(pfrag->page); 1323 } 1324 pfrag->offset += copy; 1325 } else { 1326 err = skb_zerocopy_iter_stream(sk, skb, msg, copy, uarg); 1327 if (err == -EMSGSIZE || err == -EEXIST) { 1328 tcp_mark_push(tp, skb); 1329 goto new_segment; 1330 } 1331 if (err < 0) 1332 goto do_error; 1333 copy = err; 1334 } 1335 1336 if (!copied) 1337 TCP_SKB_CB(skb)->tcp_flags &= ~TCPHDR_PSH; 1338 1339 tp->write_seq += copy; 1340 TCP_SKB_CB(skb)->end_seq += copy; 1341 tcp_skb_pcount_set(skb, 0); 1342 1343 copied += copy; 1344 if (!msg_data_left(msg)) { 1345 if (unlikely(flags & MSG_EOR)) 1346 TCP_SKB_CB(skb)->eor = 1; 1347 goto out; 1348 } 1349 1350 if (skb->len < size_goal || (flags & MSG_OOB) || unlikely(tp->repair)) 1351 continue; 1352 1353 if (forced_push(tp)) { 1354 tcp_mark_push(tp, skb); 1355 __tcp_push_pending_frames(sk, mss_now, TCP_NAGLE_PUSH); 1356 } else if (skb == tcp_send_head(sk)) 1357 tcp_push_one(sk, mss_now); 1358 continue; 1359 1360 wait_for_sndbuf: 1361 set_bit(SOCK_NOSPACE, &sk->sk_socket->flags); 1362 wait_for_memory: 1363 if (copied) 1364 tcp_push(sk, flags & ~MSG_MORE, mss_now, 1365 TCP_NAGLE_PUSH, size_goal); 1366 1367 err = sk_stream_wait_memory(sk, &timeo); 1368 if (err != 0) 1369 goto do_error; 1370 1371 mss_now = tcp_send_mss(sk, &size_goal, flags); 1372 } 1373 1374 out: 1375 if (copied) { 1376 tcp_tx_timestamp(sk, sockc.tsflags); 1377 tcp_push(sk, flags, mss_now, tp->nonagle, size_goal); 1378 } 1379 out_nopush: 1380 sock_zerocopy_put(uarg); 1381 return copied + copied_syn; 1382 1383 do_fault: 1384 if (!skb->len) { 1385 tcp_unlink_write_queue(skb, sk); 1386 /* It is the one place in all of TCP, except connection 1387 * reset, where we can be unlinking the send_head. 1388 */ 1389 if (tcp_write_queue_empty(sk)) 1390 tcp_chrono_stop(sk, TCP_CHRONO_BUSY); 1391 sk_wmem_free_skb(sk, skb); 1392 } 1393 1394 do_error: 1395 if (copied + copied_syn) 1396 goto out; 1397 out_err: 1398 sock_zerocopy_put_abort(uarg, true); 1399 err = sk_stream_error(sk, flags, err); 1400 /* make sure we wake any epoll edge trigger waiter */ 1401 if (unlikely(skb_queue_len(&sk->sk_write_queue) == 0 && 1402 err == -EAGAIN)) { 1403 sk->sk_write_space(sk); 1404 tcp_chrono_stop(sk, TCP_CHRONO_SNDBUF_LIMITED); 1405 } 1406 return err; 1407 } 1408 EXPORT_SYMBOL_GPL(tcp_sendmsg_locked); 1409 1410 int tcp_sendmsg(struct sock *sk, struct msghdr *msg, size_t size) 1411 { 1412 int ret; 1413 1414 lock_sock(sk); 1415 ret = tcp_sendmsg_locked(sk, msg, size); 1416 release_sock(sk); 1417 1418 return ret; 1419 } 1420 EXPORT_SYMBOL(tcp_sendmsg); 1421 1422 /* 1423 * Handle reading urgent data. BSD has very simple semantics for 1424 * this, no blocking and very strange errors 8) 1425 */ 1426 1427 static int tcp_recv_urg(struct sock *sk, struct msghdr *msg, int len, int flags) 1428 { 1429 struct tcp_sock *tp = tcp_sk(sk); 1430 1431 /* No URG data to read. */ 1432 if (sock_flag(sk, SOCK_URGINLINE) || !tp->urg_data || 1433 tp->urg_data == TCP_URG_READ) 1434 return -EINVAL; /* Yes this is right ! */ 1435 1436 if (sk->sk_state == TCP_CLOSE && !sock_flag(sk, SOCK_DONE)) 1437 return -ENOTCONN; 1438 1439 if (tp->urg_data & TCP_URG_VALID) { 1440 int err = 0; 1441 char c = tp->urg_data; 1442 1443 if (!(flags & MSG_PEEK)) 1444 tp->urg_data = TCP_URG_READ; 1445 1446 /* Read urgent data. */ 1447 msg->msg_flags |= MSG_OOB; 1448 1449 if (len > 0) { 1450 if (!(flags & MSG_TRUNC)) 1451 err = memcpy_to_msg(msg, &c, 1); 1452 len = 1; 1453 } else 1454 msg->msg_flags |= MSG_TRUNC; 1455 1456 return err ? -EFAULT : len; 1457 } 1458 1459 if (sk->sk_state == TCP_CLOSE || (sk->sk_shutdown & RCV_SHUTDOWN)) 1460 return 0; 1461 1462 /* Fixed the recv(..., MSG_OOB) behaviour. BSD docs and 1463 * the available implementations agree in this case: 1464 * this call should never block, independent of the 1465 * blocking state of the socket. 1466 * Mike <pall@rz.uni-karlsruhe.de> 1467 */ 1468 return -EAGAIN; 1469 } 1470 1471 static int tcp_peek_sndq(struct sock *sk, struct msghdr *msg, int len) 1472 { 1473 struct sk_buff *skb; 1474 int copied = 0, err = 0; 1475 1476 /* XXX -- need to support SO_PEEK_OFF */ 1477 1478 skb_rbtree_walk(skb, &sk->tcp_rtx_queue) { 1479 err = skb_copy_datagram_msg(skb, 0, msg, skb->len); 1480 if (err) 1481 return err; 1482 copied += skb->len; 1483 } 1484 1485 skb_queue_walk(&sk->sk_write_queue, skb) { 1486 err = skb_copy_datagram_msg(skb, 0, msg, skb->len); 1487 if (err) 1488 break; 1489 1490 copied += skb->len; 1491 } 1492 1493 return err ?: copied; 1494 } 1495 1496 /* Clean up the receive buffer for full frames taken by the user, 1497 * then send an ACK if necessary. COPIED is the number of bytes 1498 * tcp_recvmsg has given to the user so far, it speeds up the 1499 * calculation of whether or not we must ACK for the sake of 1500 * a window update. 1501 */ 1502 static void tcp_cleanup_rbuf(struct sock *sk, int copied) 1503 { 1504 struct tcp_sock *tp = tcp_sk(sk); 1505 bool time_to_ack = false; 1506 1507 struct sk_buff *skb = skb_peek(&sk->sk_receive_queue); 1508 1509 WARN(skb && !before(tp->copied_seq, TCP_SKB_CB(skb)->end_seq), 1510 "cleanup rbuf bug: copied %X seq %X rcvnxt %X\n", 1511 tp->copied_seq, TCP_SKB_CB(skb)->end_seq, tp->rcv_nxt); 1512 1513 if (inet_csk_ack_scheduled(sk)) { 1514 const struct inet_connection_sock *icsk = inet_csk(sk); 1515 /* Delayed ACKs frequently hit locked sockets during bulk 1516 * receive. */ 1517 if (icsk->icsk_ack.blocked || 1518 /* Once-per-two-segments ACK was not sent by tcp_input.c */ 1519 tp->rcv_nxt - tp->rcv_wup > icsk->icsk_ack.rcv_mss || 1520 /* 1521 * If this read emptied read buffer, we send ACK, if 1522 * connection is not bidirectional, user drained 1523 * receive buffer and there was a small segment 1524 * in queue. 1525 */ 1526 (copied > 0 && 1527 ((icsk->icsk_ack.pending & ICSK_ACK_PUSHED2) || 1528 ((icsk->icsk_ack.pending & ICSK_ACK_PUSHED) && 1529 !inet_csk_in_pingpong_mode(sk))) && 1530 !atomic_read(&sk->sk_rmem_alloc))) 1531 time_to_ack = true; 1532 } 1533 1534 /* We send an ACK if we can now advertise a non-zero window 1535 * which has been raised "significantly". 1536 * 1537 * Even if window raised up to infinity, do not send window open ACK 1538 * in states, where we will not receive more. It is useless. 1539 */ 1540 if (copied > 0 && !time_to_ack && !(sk->sk_shutdown & RCV_SHUTDOWN)) { 1541 __u32 rcv_window_now = tcp_receive_window(tp); 1542 1543 /* Optimize, __tcp_select_window() is not cheap. */ 1544 if (2*rcv_window_now <= tp->window_clamp) { 1545 __u32 new_window = __tcp_select_window(sk); 1546 1547 /* Send ACK now, if this read freed lots of space 1548 * in our buffer. Certainly, new_window is new window. 1549 * We can advertise it now, if it is not less than current one. 1550 * "Lots" means "at least twice" here. 1551 */ 1552 if (new_window && new_window >= 2 * rcv_window_now) 1553 time_to_ack = true; 1554 } 1555 } 1556 if (time_to_ack) 1557 tcp_send_ack(sk); 1558 } 1559 1560 static struct sk_buff *tcp_recv_skb(struct sock *sk, u32 seq, u32 *off) 1561 { 1562 struct sk_buff *skb; 1563 u32 offset; 1564 1565 while ((skb = skb_peek(&sk->sk_receive_queue)) != NULL) { 1566 offset = seq - TCP_SKB_CB(skb)->seq; 1567 if (unlikely(TCP_SKB_CB(skb)->tcp_flags & TCPHDR_SYN)) { 1568 pr_err_once("%s: found a SYN, please report !\n", __func__); 1569 offset--; 1570 } 1571 if (offset < skb->len || (TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN)) { 1572 *off = offset; 1573 return skb; 1574 } 1575 /* This looks weird, but this can happen if TCP collapsing 1576 * splitted a fat GRO packet, while we released socket lock 1577 * in skb_splice_bits() 1578 */ 1579 sk_eat_skb(sk, skb); 1580 } 1581 return NULL; 1582 } 1583 1584 /* 1585 * This routine provides an alternative to tcp_recvmsg() for routines 1586 * that would like to handle copying from skbuffs directly in 'sendfile' 1587 * fashion. 1588 * Note: 1589 * - It is assumed that the socket was locked by the caller. 1590 * - The routine does not block. 1591 * - At present, there is no support for reading OOB data 1592 * or for 'peeking' the socket using this routine 1593 * (although both would be easy to implement). 1594 */ 1595 int tcp_read_sock(struct sock *sk, read_descriptor_t *desc, 1596 sk_read_actor_t recv_actor) 1597 { 1598 struct sk_buff *skb; 1599 struct tcp_sock *tp = tcp_sk(sk); 1600 u32 seq = tp->copied_seq; 1601 u32 offset; 1602 int copied = 0; 1603 1604 if (sk->sk_state == TCP_LISTEN) 1605 return -ENOTCONN; 1606 while ((skb = tcp_recv_skb(sk, seq, &offset)) != NULL) { 1607 if (offset < skb->len) { 1608 int used; 1609 size_t len; 1610 1611 len = skb->len - offset; 1612 /* Stop reading if we hit a patch of urgent data */ 1613 if (tp->urg_data) { 1614 u32 urg_offset = tp->urg_seq - seq; 1615 if (urg_offset < len) 1616 len = urg_offset; 1617 if (!len) 1618 break; 1619 } 1620 used = recv_actor(desc, skb, offset, len); 1621 if (used <= 0) { 1622 if (!copied) 1623 copied = used; 1624 break; 1625 } else if (used <= len) { 1626 seq += used; 1627 copied += used; 1628 offset += used; 1629 } 1630 /* If recv_actor drops the lock (e.g. TCP splice 1631 * receive) the skb pointer might be invalid when 1632 * getting here: tcp_collapse might have deleted it 1633 * while aggregating skbs from the socket queue. 1634 */ 1635 skb = tcp_recv_skb(sk, seq - 1, &offset); 1636 if (!skb) 1637 break; 1638 /* TCP coalescing might have appended data to the skb. 1639 * Try to splice more frags 1640 */ 1641 if (offset + 1 != skb->len) 1642 continue; 1643 } 1644 if (TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN) { 1645 sk_eat_skb(sk, skb); 1646 ++seq; 1647 break; 1648 } 1649 sk_eat_skb(sk, skb); 1650 if (!desc->count) 1651 break; 1652 tp->copied_seq = seq; 1653 } 1654 tp->copied_seq = seq; 1655 1656 tcp_rcv_space_adjust(sk); 1657 1658 /* Clean up data we have read: This will do ACK frames. */ 1659 if (copied > 0) { 1660 tcp_recv_skb(sk, seq, &offset); 1661 tcp_cleanup_rbuf(sk, copied); 1662 } 1663 return copied; 1664 } 1665 EXPORT_SYMBOL(tcp_read_sock); 1666 1667 int tcp_peek_len(struct socket *sock) 1668 { 1669 return tcp_inq(sock->sk); 1670 } 1671 EXPORT_SYMBOL(tcp_peek_len); 1672 1673 /* Make sure sk_rcvbuf is big enough to satisfy SO_RCVLOWAT hint */ 1674 int tcp_set_rcvlowat(struct sock *sk, int val) 1675 { 1676 int cap; 1677 1678 if (sk->sk_userlocks & SOCK_RCVBUF_LOCK) 1679 cap = sk->sk_rcvbuf >> 1; 1680 else 1681 cap = sock_net(sk)->ipv4.sysctl_tcp_rmem[2] >> 1; 1682 val = min(val, cap); 1683 sk->sk_rcvlowat = val ? : 1; 1684 1685 /* Check if we need to signal EPOLLIN right now */ 1686 tcp_data_ready(sk); 1687 1688 if (sk->sk_userlocks & SOCK_RCVBUF_LOCK) 1689 return 0; 1690 1691 val <<= 1; 1692 if (val > sk->sk_rcvbuf) { 1693 sk->sk_rcvbuf = val; 1694 tcp_sk(sk)->window_clamp = tcp_win_from_space(sk, val); 1695 } 1696 return 0; 1697 } 1698 EXPORT_SYMBOL(tcp_set_rcvlowat); 1699 1700 #ifdef CONFIG_MMU 1701 static const struct vm_operations_struct tcp_vm_ops = { 1702 }; 1703 1704 int tcp_mmap(struct file *file, struct socket *sock, 1705 struct vm_area_struct *vma) 1706 { 1707 if (vma->vm_flags & (VM_WRITE | VM_EXEC)) 1708 return -EPERM; 1709 vma->vm_flags &= ~(VM_MAYWRITE | VM_MAYEXEC); 1710 1711 /* Instruct vm_insert_page() to not down_read(mmap_sem) */ 1712 vma->vm_flags |= VM_MIXEDMAP; 1713 1714 vma->vm_ops = &tcp_vm_ops; 1715 return 0; 1716 } 1717 EXPORT_SYMBOL(tcp_mmap); 1718 1719 static int tcp_zerocopy_receive(struct sock *sk, 1720 struct tcp_zerocopy_receive *zc) 1721 { 1722 unsigned long address = (unsigned long)zc->address; 1723 const skb_frag_t *frags = NULL; 1724 u32 length = 0, seq, offset; 1725 struct vm_area_struct *vma; 1726 struct sk_buff *skb = NULL; 1727 struct tcp_sock *tp; 1728 int inq; 1729 int ret; 1730 1731 if (address & (PAGE_SIZE - 1) || address != zc->address) 1732 return -EINVAL; 1733 1734 if (sk->sk_state == TCP_LISTEN) 1735 return -ENOTCONN; 1736 1737 sock_rps_record_flow(sk); 1738 1739 down_read(¤t->mm->mmap_sem); 1740 1741 ret = -EINVAL; 1742 vma = find_vma(current->mm, address); 1743 if (!vma || vma->vm_start > address || vma->vm_ops != &tcp_vm_ops) 1744 goto out; 1745 zc->length = min_t(unsigned long, zc->length, vma->vm_end - address); 1746 1747 tp = tcp_sk(sk); 1748 seq = tp->copied_seq; 1749 inq = tcp_inq(sk); 1750 zc->length = min_t(u32, zc->length, inq); 1751 zc->length &= ~(PAGE_SIZE - 1); 1752 if (zc->length) { 1753 zap_page_range(vma, address, zc->length); 1754 zc->recv_skip_hint = 0; 1755 } else { 1756 zc->recv_skip_hint = inq; 1757 } 1758 ret = 0; 1759 while (length + PAGE_SIZE <= zc->length) { 1760 if (zc->recv_skip_hint < PAGE_SIZE) { 1761 if (skb) { 1762 skb = skb->next; 1763 offset = seq - TCP_SKB_CB(skb)->seq; 1764 } else { 1765 skb = tcp_recv_skb(sk, seq, &offset); 1766 } 1767 1768 zc->recv_skip_hint = skb->len - offset; 1769 offset -= skb_headlen(skb); 1770 if ((int)offset < 0 || skb_has_frag_list(skb)) 1771 break; 1772 frags = skb_shinfo(skb)->frags; 1773 while (offset) { 1774 if (frags->size > offset) 1775 goto out; 1776 offset -= frags->size; 1777 frags++; 1778 } 1779 } 1780 if (frags->size != PAGE_SIZE || frags->page_offset) { 1781 int remaining = zc->recv_skip_hint; 1782 1783 while (remaining && (frags->size != PAGE_SIZE || 1784 frags->page_offset)) { 1785 remaining -= frags->size; 1786 frags++; 1787 } 1788 zc->recv_skip_hint -= remaining; 1789 break; 1790 } 1791 ret = vm_insert_page(vma, address + length, 1792 skb_frag_page(frags)); 1793 if (ret) 1794 break; 1795 length += PAGE_SIZE; 1796 seq += PAGE_SIZE; 1797 zc->recv_skip_hint -= PAGE_SIZE; 1798 frags++; 1799 } 1800 out: 1801 up_read(¤t->mm->mmap_sem); 1802 if (length) { 1803 tp->copied_seq = seq; 1804 tcp_rcv_space_adjust(sk); 1805 1806 /* Clean up data we have read: This will do ACK frames. */ 1807 tcp_recv_skb(sk, seq, &offset); 1808 tcp_cleanup_rbuf(sk, length); 1809 ret = 0; 1810 if (length == zc->length) 1811 zc->recv_skip_hint = 0; 1812 } else { 1813 if (!zc->recv_skip_hint && sock_flag(sk, SOCK_DONE)) 1814 ret = -EIO; 1815 } 1816 zc->length = length; 1817 return ret; 1818 } 1819 #endif 1820 1821 static void tcp_update_recv_tstamps(struct sk_buff *skb, 1822 struct scm_timestamping_internal *tss) 1823 { 1824 if (skb->tstamp) 1825 tss->ts[0] = ktime_to_timespec64(skb->tstamp); 1826 else 1827 tss->ts[0] = (struct timespec64) {0}; 1828 1829 if (skb_hwtstamps(skb)->hwtstamp) 1830 tss->ts[2] = ktime_to_timespec64(skb_hwtstamps(skb)->hwtstamp); 1831 else 1832 tss->ts[2] = (struct timespec64) {0}; 1833 } 1834 1835 /* Similar to __sock_recv_timestamp, but does not require an skb */ 1836 static void tcp_recv_timestamp(struct msghdr *msg, const struct sock *sk, 1837 struct scm_timestamping_internal *tss) 1838 { 1839 int new_tstamp = sock_flag(sk, SOCK_TSTAMP_NEW); 1840 bool has_timestamping = false; 1841 1842 if (tss->ts[0].tv_sec || tss->ts[0].tv_nsec) { 1843 if (sock_flag(sk, SOCK_RCVTSTAMP)) { 1844 if (sock_flag(sk, SOCK_RCVTSTAMPNS)) { 1845 if (new_tstamp) { 1846 struct __kernel_timespec kts = {tss->ts[0].tv_sec, tss->ts[0].tv_nsec}; 1847 1848 put_cmsg(msg, SOL_SOCKET, SO_TIMESTAMPNS_NEW, 1849 sizeof(kts), &kts); 1850 } else { 1851 struct timespec ts_old = timespec64_to_timespec(tss->ts[0]); 1852 1853 put_cmsg(msg, SOL_SOCKET, SO_TIMESTAMPNS_OLD, 1854 sizeof(ts_old), &ts_old); 1855 } 1856 } else { 1857 if (new_tstamp) { 1858 struct __kernel_sock_timeval stv; 1859 1860 stv.tv_sec = tss->ts[0].tv_sec; 1861 stv.tv_usec = tss->ts[0].tv_nsec / 1000; 1862 put_cmsg(msg, SOL_SOCKET, SO_TIMESTAMP_NEW, 1863 sizeof(stv), &stv); 1864 } else { 1865 struct __kernel_old_timeval tv; 1866 1867 tv.tv_sec = tss->ts[0].tv_sec; 1868 tv.tv_usec = tss->ts[0].tv_nsec / 1000; 1869 put_cmsg(msg, SOL_SOCKET, SO_TIMESTAMP_OLD, 1870 sizeof(tv), &tv); 1871 } 1872 } 1873 } 1874 1875 if (sk->sk_tsflags & SOF_TIMESTAMPING_SOFTWARE) 1876 has_timestamping = true; 1877 else 1878 tss->ts[0] = (struct timespec64) {0}; 1879 } 1880 1881 if (tss->ts[2].tv_sec || tss->ts[2].tv_nsec) { 1882 if (sk->sk_tsflags & SOF_TIMESTAMPING_RAW_HARDWARE) 1883 has_timestamping = true; 1884 else 1885 tss->ts[2] = (struct timespec64) {0}; 1886 } 1887 1888 if (has_timestamping) { 1889 tss->ts[1] = (struct timespec64) {0}; 1890 if (sock_flag(sk, SOCK_TSTAMP_NEW)) 1891 put_cmsg_scm_timestamping64(msg, tss); 1892 else 1893 put_cmsg_scm_timestamping(msg, tss); 1894 } 1895 } 1896 1897 static int tcp_inq_hint(struct sock *sk) 1898 { 1899 const struct tcp_sock *tp = tcp_sk(sk); 1900 u32 copied_seq = READ_ONCE(tp->copied_seq); 1901 u32 rcv_nxt = READ_ONCE(tp->rcv_nxt); 1902 int inq; 1903 1904 inq = rcv_nxt - copied_seq; 1905 if (unlikely(inq < 0 || copied_seq != READ_ONCE(tp->copied_seq))) { 1906 lock_sock(sk); 1907 inq = tp->rcv_nxt - tp->copied_seq; 1908 release_sock(sk); 1909 } 1910 /* After receiving a FIN, tell the user-space to continue reading 1911 * by returning a non-zero inq. 1912 */ 1913 if (inq == 0 && sock_flag(sk, SOCK_DONE)) 1914 inq = 1; 1915 return inq; 1916 } 1917 1918 /* 1919 * This routine copies from a sock struct into the user buffer. 1920 * 1921 * Technical note: in 2.3 we work on _locked_ socket, so that 1922 * tricks with *seq access order and skb->users are not required. 1923 * Probably, code can be easily improved even more. 1924 */ 1925 1926 int tcp_recvmsg(struct sock *sk, struct msghdr *msg, size_t len, int nonblock, 1927 int flags, int *addr_len) 1928 { 1929 struct tcp_sock *tp = tcp_sk(sk); 1930 int copied = 0; 1931 u32 peek_seq; 1932 u32 *seq; 1933 unsigned long used; 1934 int err, inq; 1935 int target; /* Read at least this many bytes */ 1936 long timeo; 1937 struct sk_buff *skb, *last; 1938 u32 urg_hole = 0; 1939 struct scm_timestamping_internal tss; 1940 bool has_tss = false; 1941 bool has_cmsg; 1942 1943 if (unlikely(flags & MSG_ERRQUEUE)) 1944 return inet_recv_error(sk, msg, len, addr_len); 1945 1946 if (sk_can_busy_loop(sk) && skb_queue_empty(&sk->sk_receive_queue) && 1947 (sk->sk_state == TCP_ESTABLISHED)) 1948 sk_busy_loop(sk, nonblock); 1949 1950 lock_sock(sk); 1951 1952 err = -ENOTCONN; 1953 if (sk->sk_state == TCP_LISTEN) 1954 goto out; 1955 1956 has_cmsg = tp->recvmsg_inq; 1957 timeo = sock_rcvtimeo(sk, nonblock); 1958 1959 /* Urgent data needs to be handled specially. */ 1960 if (flags & MSG_OOB) 1961 goto recv_urg; 1962 1963 if (unlikely(tp->repair)) { 1964 err = -EPERM; 1965 if (!(flags & MSG_PEEK)) 1966 goto out; 1967 1968 if (tp->repair_queue == TCP_SEND_QUEUE) 1969 goto recv_sndq; 1970 1971 err = -EINVAL; 1972 if (tp->repair_queue == TCP_NO_QUEUE) 1973 goto out; 1974 1975 /* 'common' recv queue MSG_PEEK-ing */ 1976 } 1977 1978 seq = &tp->copied_seq; 1979 if (flags & MSG_PEEK) { 1980 peek_seq = tp->copied_seq; 1981 seq = &peek_seq; 1982 } 1983 1984 target = sock_rcvlowat(sk, flags & MSG_WAITALL, len); 1985 1986 do { 1987 u32 offset; 1988 1989 /* Are we at urgent data? Stop if we have read anything or have SIGURG pending. */ 1990 if (tp->urg_data && tp->urg_seq == *seq) { 1991 if (copied) 1992 break; 1993 if (signal_pending(current)) { 1994 copied = timeo ? sock_intr_errno(timeo) : -EAGAIN; 1995 break; 1996 } 1997 } 1998 1999 /* Next get a buffer. */ 2000 2001 last = skb_peek_tail(&sk->sk_receive_queue); 2002 skb_queue_walk(&sk->sk_receive_queue, skb) { 2003 last = skb; 2004 /* Now that we have two receive queues this 2005 * shouldn't happen. 2006 */ 2007 if (WARN(before(*seq, TCP_SKB_CB(skb)->seq), 2008 "TCP recvmsg seq # bug: copied %X, seq %X, rcvnxt %X, fl %X\n", 2009 *seq, TCP_SKB_CB(skb)->seq, tp->rcv_nxt, 2010 flags)) 2011 break; 2012 2013 offset = *seq - TCP_SKB_CB(skb)->seq; 2014 if (unlikely(TCP_SKB_CB(skb)->tcp_flags & TCPHDR_SYN)) { 2015 pr_err_once("%s: found a SYN, please report !\n", __func__); 2016 offset--; 2017 } 2018 if (offset < skb->len) 2019 goto found_ok_skb; 2020 if (TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN) 2021 goto found_fin_ok; 2022 WARN(!(flags & MSG_PEEK), 2023 "TCP recvmsg seq # bug 2: copied %X, seq %X, rcvnxt %X, fl %X\n", 2024 *seq, TCP_SKB_CB(skb)->seq, tp->rcv_nxt, flags); 2025 } 2026 2027 /* Well, if we have backlog, try to process it now yet. */ 2028 2029 if (copied >= target && !sk->sk_backlog.tail) 2030 break; 2031 2032 if (copied) { 2033 if (sk->sk_err || 2034 sk->sk_state == TCP_CLOSE || 2035 (sk->sk_shutdown & RCV_SHUTDOWN) || 2036 !timeo || 2037 signal_pending(current)) 2038 break; 2039 } else { 2040 if (sock_flag(sk, SOCK_DONE)) 2041 break; 2042 2043 if (sk->sk_err) { 2044 copied = sock_error(sk); 2045 break; 2046 } 2047 2048 if (sk->sk_shutdown & RCV_SHUTDOWN) 2049 break; 2050 2051 if (sk->sk_state == TCP_CLOSE) { 2052 /* This occurs when user tries to read 2053 * from never connected socket. 2054 */ 2055 copied = -ENOTCONN; 2056 break; 2057 } 2058 2059 if (!timeo) { 2060 copied = -EAGAIN; 2061 break; 2062 } 2063 2064 if (signal_pending(current)) { 2065 copied = sock_intr_errno(timeo); 2066 break; 2067 } 2068 } 2069 2070 tcp_cleanup_rbuf(sk, copied); 2071 2072 if (copied >= target) { 2073 /* Do not sleep, just process backlog. */ 2074 release_sock(sk); 2075 lock_sock(sk); 2076 } else { 2077 sk_wait_data(sk, &timeo, last); 2078 } 2079 2080 if ((flags & MSG_PEEK) && 2081 (peek_seq - copied - urg_hole != tp->copied_seq)) { 2082 net_dbg_ratelimited("TCP(%s:%d): Application bug, race in MSG_PEEK\n", 2083 current->comm, 2084 task_pid_nr(current)); 2085 peek_seq = tp->copied_seq; 2086 } 2087 continue; 2088 2089 found_ok_skb: 2090 /* Ok so how much can we use? */ 2091 used = skb->len - offset; 2092 if (len < used) 2093 used = len; 2094 2095 /* Do we have urgent data here? */ 2096 if (tp->urg_data) { 2097 u32 urg_offset = tp->urg_seq - *seq; 2098 if (urg_offset < used) { 2099 if (!urg_offset) { 2100 if (!sock_flag(sk, SOCK_URGINLINE)) { 2101 ++*seq; 2102 urg_hole++; 2103 offset++; 2104 used--; 2105 if (!used) 2106 goto skip_copy; 2107 } 2108 } else 2109 used = urg_offset; 2110 } 2111 } 2112 2113 if (!(flags & MSG_TRUNC)) { 2114 err = skb_copy_datagram_msg(skb, offset, msg, used); 2115 if (err) { 2116 /* Exception. Bailout! */ 2117 if (!copied) 2118 copied = -EFAULT; 2119 break; 2120 } 2121 } 2122 2123 *seq += used; 2124 copied += used; 2125 len -= used; 2126 2127 tcp_rcv_space_adjust(sk); 2128 2129 skip_copy: 2130 if (tp->urg_data && after(tp->copied_seq, tp->urg_seq)) { 2131 tp->urg_data = 0; 2132 tcp_fast_path_check(sk); 2133 } 2134 if (used + offset < skb->len) 2135 continue; 2136 2137 if (TCP_SKB_CB(skb)->has_rxtstamp) { 2138 tcp_update_recv_tstamps(skb, &tss); 2139 has_tss = true; 2140 has_cmsg = true; 2141 } 2142 if (TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN) 2143 goto found_fin_ok; 2144 if (!(flags & MSG_PEEK)) 2145 sk_eat_skb(sk, skb); 2146 continue; 2147 2148 found_fin_ok: 2149 /* Process the FIN. */ 2150 ++*seq; 2151 if (!(flags & MSG_PEEK)) 2152 sk_eat_skb(sk, skb); 2153 break; 2154 } while (len > 0); 2155 2156 /* According to UNIX98, msg_name/msg_namelen are ignored 2157 * on connected socket. I was just happy when found this 8) --ANK 2158 */ 2159 2160 /* Clean up data we have read: This will do ACK frames. */ 2161 tcp_cleanup_rbuf(sk, copied); 2162 2163 release_sock(sk); 2164 2165 if (has_cmsg) { 2166 if (has_tss) 2167 tcp_recv_timestamp(msg, sk, &tss); 2168 if (tp->recvmsg_inq) { 2169 inq = tcp_inq_hint(sk); 2170 put_cmsg(msg, SOL_TCP, TCP_CM_INQ, sizeof(inq), &inq); 2171 } 2172 } 2173 2174 return copied; 2175 2176 out: 2177 release_sock(sk); 2178 return err; 2179 2180 recv_urg: 2181 err = tcp_recv_urg(sk, msg, len, flags); 2182 goto out; 2183 2184 recv_sndq: 2185 err = tcp_peek_sndq(sk, msg, len); 2186 goto out; 2187 } 2188 EXPORT_SYMBOL(tcp_recvmsg); 2189 2190 void tcp_set_state(struct sock *sk, int state) 2191 { 2192 int oldstate = sk->sk_state; 2193 2194 /* We defined a new enum for TCP states that are exported in BPF 2195 * so as not force the internal TCP states to be frozen. The 2196 * following checks will detect if an internal state value ever 2197 * differs from the BPF value. If this ever happens, then we will 2198 * need to remap the internal value to the BPF value before calling 2199 * tcp_call_bpf_2arg. 2200 */ 2201 BUILD_BUG_ON((int)BPF_TCP_ESTABLISHED != (int)TCP_ESTABLISHED); 2202 BUILD_BUG_ON((int)BPF_TCP_SYN_SENT != (int)TCP_SYN_SENT); 2203 BUILD_BUG_ON((int)BPF_TCP_SYN_RECV != (int)TCP_SYN_RECV); 2204 BUILD_BUG_ON((int)BPF_TCP_FIN_WAIT1 != (int)TCP_FIN_WAIT1); 2205 BUILD_BUG_ON((int)BPF_TCP_FIN_WAIT2 != (int)TCP_FIN_WAIT2); 2206 BUILD_BUG_ON((int)BPF_TCP_TIME_WAIT != (int)TCP_TIME_WAIT); 2207 BUILD_BUG_ON((int)BPF_TCP_CLOSE != (int)TCP_CLOSE); 2208 BUILD_BUG_ON((int)BPF_TCP_CLOSE_WAIT != (int)TCP_CLOSE_WAIT); 2209 BUILD_BUG_ON((int)BPF_TCP_LAST_ACK != (int)TCP_LAST_ACK); 2210 BUILD_BUG_ON((int)BPF_TCP_LISTEN != (int)TCP_LISTEN); 2211 BUILD_BUG_ON((int)BPF_TCP_CLOSING != (int)TCP_CLOSING); 2212 BUILD_BUG_ON((int)BPF_TCP_NEW_SYN_RECV != (int)TCP_NEW_SYN_RECV); 2213 BUILD_BUG_ON((int)BPF_TCP_MAX_STATES != (int)TCP_MAX_STATES); 2214 2215 if (BPF_SOCK_OPS_TEST_FLAG(tcp_sk(sk), BPF_SOCK_OPS_STATE_CB_FLAG)) 2216 tcp_call_bpf_2arg(sk, BPF_SOCK_OPS_STATE_CB, oldstate, state); 2217 2218 switch (state) { 2219 case TCP_ESTABLISHED: 2220 if (oldstate != TCP_ESTABLISHED) 2221 TCP_INC_STATS(sock_net(sk), TCP_MIB_CURRESTAB); 2222 break; 2223 2224 case TCP_CLOSE: 2225 if (oldstate == TCP_CLOSE_WAIT || oldstate == TCP_ESTABLISHED) 2226 TCP_INC_STATS(sock_net(sk), TCP_MIB_ESTABRESETS); 2227 2228 sk->sk_prot->unhash(sk); 2229 if (inet_csk(sk)->icsk_bind_hash && 2230 !(sk->sk_userlocks & SOCK_BINDPORT_LOCK)) 2231 inet_put_port(sk); 2232 /* fall through */ 2233 default: 2234 if (oldstate == TCP_ESTABLISHED) 2235 TCP_DEC_STATS(sock_net(sk), TCP_MIB_CURRESTAB); 2236 } 2237 2238 /* Change state AFTER socket is unhashed to avoid closed 2239 * socket sitting in hash tables. 2240 */ 2241 inet_sk_state_store(sk, state); 2242 } 2243 EXPORT_SYMBOL_GPL(tcp_set_state); 2244 2245 /* 2246 * State processing on a close. This implements the state shift for 2247 * sending our FIN frame. Note that we only send a FIN for some 2248 * states. A shutdown() may have already sent the FIN, or we may be 2249 * closed. 2250 */ 2251 2252 static const unsigned char new_state[16] = { 2253 /* current state: new state: action: */ 2254 [0 /* (Invalid) */] = TCP_CLOSE, 2255 [TCP_ESTABLISHED] = TCP_FIN_WAIT1 | TCP_ACTION_FIN, 2256 [TCP_SYN_SENT] = TCP_CLOSE, 2257 [TCP_SYN_RECV] = TCP_FIN_WAIT1 | TCP_ACTION_FIN, 2258 [TCP_FIN_WAIT1] = TCP_FIN_WAIT1, 2259 [TCP_FIN_WAIT2] = TCP_FIN_WAIT2, 2260 [TCP_TIME_WAIT] = TCP_CLOSE, 2261 [TCP_CLOSE] = TCP_CLOSE, 2262 [TCP_CLOSE_WAIT] = TCP_LAST_ACK | TCP_ACTION_FIN, 2263 [TCP_LAST_ACK] = TCP_LAST_ACK, 2264 [TCP_LISTEN] = TCP_CLOSE, 2265 [TCP_CLOSING] = TCP_CLOSING, 2266 [TCP_NEW_SYN_RECV] = TCP_CLOSE, /* should not happen ! */ 2267 }; 2268 2269 static int tcp_close_state(struct sock *sk) 2270 { 2271 int next = (int)new_state[sk->sk_state]; 2272 int ns = next & TCP_STATE_MASK; 2273 2274 tcp_set_state(sk, ns); 2275 2276 return next & TCP_ACTION_FIN; 2277 } 2278 2279 /* 2280 * Shutdown the sending side of a connection. Much like close except 2281 * that we don't receive shut down or sock_set_flag(sk, SOCK_DEAD). 2282 */ 2283 2284 void tcp_shutdown(struct sock *sk, int how) 2285 { 2286 /* We need to grab some memory, and put together a FIN, 2287 * and then put it into the queue to be sent. 2288 * Tim MacKenzie(tym@dibbler.cs.monash.edu.au) 4 Dec '92. 2289 */ 2290 if (!(how & SEND_SHUTDOWN)) 2291 return; 2292 2293 /* If we've already sent a FIN, or it's a closed state, skip this. */ 2294 if ((1 << sk->sk_state) & 2295 (TCPF_ESTABLISHED | TCPF_SYN_SENT | 2296 TCPF_SYN_RECV | TCPF_CLOSE_WAIT)) { 2297 /* Clear out any half completed packets. FIN if needed. */ 2298 if (tcp_close_state(sk)) 2299 tcp_send_fin(sk); 2300 } 2301 } 2302 EXPORT_SYMBOL(tcp_shutdown); 2303 2304 bool tcp_check_oom(struct sock *sk, int shift) 2305 { 2306 bool too_many_orphans, out_of_socket_memory; 2307 2308 too_many_orphans = tcp_too_many_orphans(sk, shift); 2309 out_of_socket_memory = tcp_out_of_memory(sk); 2310 2311 if (too_many_orphans) 2312 net_info_ratelimited("too many orphaned sockets\n"); 2313 if (out_of_socket_memory) 2314 net_info_ratelimited("out of memory -- consider tuning tcp_mem\n"); 2315 return too_many_orphans || out_of_socket_memory; 2316 } 2317 2318 void tcp_close(struct sock *sk, long timeout) 2319 { 2320 struct sk_buff *skb; 2321 int data_was_unread = 0; 2322 int state; 2323 2324 lock_sock(sk); 2325 sk->sk_shutdown = SHUTDOWN_MASK; 2326 2327 if (sk->sk_state == TCP_LISTEN) { 2328 tcp_set_state(sk, TCP_CLOSE); 2329 2330 /* Special case. */ 2331 inet_csk_listen_stop(sk); 2332 2333 goto adjudge_to_death; 2334 } 2335 2336 /* We need to flush the recv. buffs. We do this only on the 2337 * descriptor close, not protocol-sourced closes, because the 2338 * reader process may not have drained the data yet! 2339 */ 2340 while ((skb = __skb_dequeue(&sk->sk_receive_queue)) != NULL) { 2341 u32 len = TCP_SKB_CB(skb)->end_seq - TCP_SKB_CB(skb)->seq; 2342 2343 if (TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN) 2344 len--; 2345 data_was_unread += len; 2346 __kfree_skb(skb); 2347 } 2348 2349 sk_mem_reclaim(sk); 2350 2351 /* If socket has been already reset (e.g. in tcp_reset()) - kill it. */ 2352 if (sk->sk_state == TCP_CLOSE) 2353 goto adjudge_to_death; 2354 2355 /* As outlined in RFC 2525, section 2.17, we send a RST here because 2356 * data was lost. To witness the awful effects of the old behavior of 2357 * always doing a FIN, run an older 2.1.x kernel or 2.0.x, start a bulk 2358 * GET in an FTP client, suspend the process, wait for the client to 2359 * advertise a zero window, then kill -9 the FTP client, wheee... 2360 * Note: timeout is always zero in such a case. 2361 */ 2362 if (unlikely(tcp_sk(sk)->repair)) { 2363 sk->sk_prot->disconnect(sk, 0); 2364 } else if (data_was_unread) { 2365 /* Unread data was tossed, zap the connection. */ 2366 NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPABORTONCLOSE); 2367 tcp_set_state(sk, TCP_CLOSE); 2368 tcp_send_active_reset(sk, sk->sk_allocation); 2369 } else if (sock_flag(sk, SOCK_LINGER) && !sk->sk_lingertime) { 2370 /* Check zero linger _after_ checking for unread data. */ 2371 sk->sk_prot->disconnect(sk, 0); 2372 NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPABORTONDATA); 2373 } else if (tcp_close_state(sk)) { 2374 /* We FIN if the application ate all the data before 2375 * zapping the connection. 2376 */ 2377 2378 /* RED-PEN. Formally speaking, we have broken TCP state 2379 * machine. State transitions: 2380 * 2381 * TCP_ESTABLISHED -> TCP_FIN_WAIT1 2382 * TCP_SYN_RECV -> TCP_FIN_WAIT1 (forget it, it's impossible) 2383 * TCP_CLOSE_WAIT -> TCP_LAST_ACK 2384 * 2385 * are legal only when FIN has been sent (i.e. in window), 2386 * rather than queued out of window. Purists blame. 2387 * 2388 * F.e. "RFC state" is ESTABLISHED, 2389 * if Linux state is FIN-WAIT-1, but FIN is still not sent. 2390 * 2391 * The visible declinations are that sometimes 2392 * we enter time-wait state, when it is not required really 2393 * (harmless), do not send active resets, when they are 2394 * required by specs (TCP_ESTABLISHED, TCP_CLOSE_WAIT, when 2395 * they look as CLOSING or LAST_ACK for Linux) 2396 * Probably, I missed some more holelets. 2397 * --ANK 2398 * XXX (TFO) - To start off we don't support SYN+ACK+FIN 2399 * in a single packet! (May consider it later but will 2400 * probably need API support or TCP_CORK SYN-ACK until 2401 * data is written and socket is closed.) 2402 */ 2403 tcp_send_fin(sk); 2404 } 2405 2406 sk_stream_wait_close(sk, timeout); 2407 2408 adjudge_to_death: 2409 state = sk->sk_state; 2410 sock_hold(sk); 2411 sock_orphan(sk); 2412 2413 local_bh_disable(); 2414 bh_lock_sock(sk); 2415 /* remove backlog if any, without releasing ownership. */ 2416 __release_sock(sk); 2417 2418 percpu_counter_inc(sk->sk_prot->orphan_count); 2419 2420 /* Have we already been destroyed by a softirq or backlog? */ 2421 if (state != TCP_CLOSE && sk->sk_state == TCP_CLOSE) 2422 goto out; 2423 2424 /* This is a (useful) BSD violating of the RFC. There is a 2425 * problem with TCP as specified in that the other end could 2426 * keep a socket open forever with no application left this end. 2427 * We use a 1 minute timeout (about the same as BSD) then kill 2428 * our end. If they send after that then tough - BUT: long enough 2429 * that we won't make the old 4*rto = almost no time - whoops 2430 * reset mistake. 2431 * 2432 * Nope, it was not mistake. It is really desired behaviour 2433 * f.e. on http servers, when such sockets are useless, but 2434 * consume significant resources. Let's do it with special 2435 * linger2 option. --ANK 2436 */ 2437 2438 if (sk->sk_state == TCP_FIN_WAIT2) { 2439 struct tcp_sock *tp = tcp_sk(sk); 2440 if (tp->linger2 < 0) { 2441 tcp_set_state(sk, TCP_CLOSE); 2442 tcp_send_active_reset(sk, GFP_ATOMIC); 2443 __NET_INC_STATS(sock_net(sk), 2444 LINUX_MIB_TCPABORTONLINGER); 2445 } else { 2446 const int tmo = tcp_fin_time(sk); 2447 2448 if (tmo > TCP_TIMEWAIT_LEN) { 2449 inet_csk_reset_keepalive_timer(sk, 2450 tmo - TCP_TIMEWAIT_LEN); 2451 } else { 2452 tcp_time_wait(sk, TCP_FIN_WAIT2, tmo); 2453 goto out; 2454 } 2455 } 2456 } 2457 if (sk->sk_state != TCP_CLOSE) { 2458 sk_mem_reclaim(sk); 2459 if (tcp_check_oom(sk, 0)) { 2460 tcp_set_state(sk, TCP_CLOSE); 2461 tcp_send_active_reset(sk, GFP_ATOMIC); 2462 __NET_INC_STATS(sock_net(sk), 2463 LINUX_MIB_TCPABORTONMEMORY); 2464 } else if (!check_net(sock_net(sk))) { 2465 /* Not possible to send reset; just close */ 2466 tcp_set_state(sk, TCP_CLOSE); 2467 } 2468 } 2469 2470 if (sk->sk_state == TCP_CLOSE) { 2471 struct request_sock *req = tcp_sk(sk)->fastopen_rsk; 2472 /* We could get here with a non-NULL req if the socket is 2473 * aborted (e.g., closed with unread data) before 3WHS 2474 * finishes. 2475 */ 2476 if (req) 2477 reqsk_fastopen_remove(sk, req, false); 2478 inet_csk_destroy_sock(sk); 2479 } 2480 /* Otherwise, socket is reprieved until protocol close. */ 2481 2482 out: 2483 bh_unlock_sock(sk); 2484 local_bh_enable(); 2485 release_sock(sk); 2486 sock_put(sk); 2487 } 2488 EXPORT_SYMBOL(tcp_close); 2489 2490 /* These states need RST on ABORT according to RFC793 */ 2491 2492 static inline bool tcp_need_reset(int state) 2493 { 2494 return (1 << state) & 2495 (TCPF_ESTABLISHED | TCPF_CLOSE_WAIT | TCPF_FIN_WAIT1 | 2496 TCPF_FIN_WAIT2 | TCPF_SYN_RECV); 2497 } 2498 2499 static void tcp_rtx_queue_purge(struct sock *sk) 2500 { 2501 struct rb_node *p = rb_first(&sk->tcp_rtx_queue); 2502 2503 while (p) { 2504 struct sk_buff *skb = rb_to_skb(p); 2505 2506 p = rb_next(p); 2507 /* Since we are deleting whole queue, no need to 2508 * list_del(&skb->tcp_tsorted_anchor) 2509 */ 2510 tcp_rtx_queue_unlink(skb, sk); 2511 sk_wmem_free_skb(sk, skb); 2512 } 2513 } 2514 2515 void tcp_write_queue_purge(struct sock *sk) 2516 { 2517 struct sk_buff *skb; 2518 2519 tcp_chrono_stop(sk, TCP_CHRONO_BUSY); 2520 while ((skb = __skb_dequeue(&sk->sk_write_queue)) != NULL) { 2521 tcp_skb_tsorted_anchor_cleanup(skb); 2522 sk_wmem_free_skb(sk, skb); 2523 } 2524 tcp_rtx_queue_purge(sk); 2525 skb = sk->sk_tx_skb_cache; 2526 if (skb) { 2527 __kfree_skb(skb); 2528 sk->sk_tx_skb_cache = NULL; 2529 } 2530 INIT_LIST_HEAD(&tcp_sk(sk)->tsorted_sent_queue); 2531 sk_mem_reclaim(sk); 2532 tcp_clear_all_retrans_hints(tcp_sk(sk)); 2533 tcp_sk(sk)->packets_out = 0; 2534 inet_csk(sk)->icsk_backoff = 0; 2535 } 2536 2537 int tcp_disconnect(struct sock *sk, int flags) 2538 { 2539 struct inet_sock *inet = inet_sk(sk); 2540 struct inet_connection_sock *icsk = inet_csk(sk); 2541 struct tcp_sock *tp = tcp_sk(sk); 2542 int old_state = sk->sk_state; 2543 2544 if (old_state != TCP_CLOSE) 2545 tcp_set_state(sk, TCP_CLOSE); 2546 2547 /* ABORT function of RFC793 */ 2548 if (old_state == TCP_LISTEN) { 2549 inet_csk_listen_stop(sk); 2550 } else if (unlikely(tp->repair)) { 2551 sk->sk_err = ECONNABORTED; 2552 } else if (tcp_need_reset(old_state) || 2553 (tp->snd_nxt != tp->write_seq && 2554 (1 << old_state) & (TCPF_CLOSING | TCPF_LAST_ACK))) { 2555 /* The last check adjusts for discrepancy of Linux wrt. RFC 2556 * states 2557 */ 2558 tcp_send_active_reset(sk, gfp_any()); 2559 sk->sk_err = ECONNRESET; 2560 } else if (old_state == TCP_SYN_SENT) 2561 sk->sk_err = ECONNRESET; 2562 2563 tcp_clear_xmit_timers(sk); 2564 __skb_queue_purge(&sk->sk_receive_queue); 2565 if (sk->sk_rx_skb_cache) { 2566 __kfree_skb(sk->sk_rx_skb_cache); 2567 sk->sk_rx_skb_cache = NULL; 2568 } 2569 tp->copied_seq = tp->rcv_nxt; 2570 tp->urg_data = 0; 2571 tcp_write_queue_purge(sk); 2572 tcp_fastopen_active_disable_ofo_check(sk); 2573 skb_rbtree_purge(&tp->out_of_order_queue); 2574 2575 inet->inet_dport = 0; 2576 2577 if (!(sk->sk_userlocks & SOCK_BINDADDR_LOCK)) 2578 inet_reset_saddr(sk); 2579 2580 sk->sk_shutdown = 0; 2581 sock_reset_flag(sk, SOCK_DONE); 2582 tp->srtt_us = 0; 2583 tp->mdev_us = jiffies_to_usecs(TCP_TIMEOUT_INIT); 2584 tp->rcv_rtt_last_tsecr = 0; 2585 tp->write_seq += tp->max_window + 2; 2586 if (tp->write_seq == 0) 2587 tp->write_seq = 1; 2588 icsk->icsk_backoff = 0; 2589 tp->snd_cwnd = 2; 2590 icsk->icsk_probes_out = 0; 2591 icsk->icsk_rto = TCP_TIMEOUT_INIT; 2592 tp->snd_ssthresh = TCP_INFINITE_SSTHRESH; 2593 tp->snd_cwnd = TCP_INIT_CWND; 2594 tp->snd_cwnd_cnt = 0; 2595 tp->window_clamp = 0; 2596 tp->delivered_ce = 0; 2597 tcp_set_ca_state(sk, TCP_CA_Open); 2598 tp->is_sack_reneg = 0; 2599 tcp_clear_retrans(tp); 2600 inet_csk_delack_init(sk); 2601 /* Initialize rcv_mss to TCP_MIN_MSS to avoid division by 0 2602 * issue in __tcp_select_window() 2603 */ 2604 icsk->icsk_ack.rcv_mss = TCP_MIN_MSS; 2605 memset(&tp->rx_opt, 0, sizeof(tp->rx_opt)); 2606 __sk_dst_reset(sk); 2607 dst_release(sk->sk_rx_dst); 2608 sk->sk_rx_dst = NULL; 2609 tcp_saved_syn_free(tp); 2610 tp->compressed_ack = 0; 2611 tp->bytes_sent = 0; 2612 tp->bytes_retrans = 0; 2613 tp->duplicate_sack[0].start_seq = 0; 2614 tp->duplicate_sack[0].end_seq = 0; 2615 tp->dsack_dups = 0; 2616 tp->reord_seen = 0; 2617 tp->retrans_out = 0; 2618 tp->sacked_out = 0; 2619 tp->tlp_high_seq = 0; 2620 tp->last_oow_ack_time = 0; 2621 /* There's a bubble in the pipe until at least the first ACK. */ 2622 tp->app_limited = ~0U; 2623 tp->rack.mstamp = 0; 2624 tp->rack.advanced = 0; 2625 tp->rack.reo_wnd_steps = 1; 2626 tp->rack.last_delivered = 0; 2627 tp->rack.reo_wnd_persist = 0; 2628 tp->rack.dsack_seen = 0; 2629 tp->syn_data_acked = 0; 2630 tp->rx_opt.saw_tstamp = 0; 2631 tp->rx_opt.dsack = 0; 2632 tp->rx_opt.num_sacks = 0; 2633 2634 2635 /* Clean up fastopen related fields */ 2636 tcp_free_fastopen_req(tp); 2637 inet->defer_connect = 0; 2638 2639 WARN_ON(inet->inet_num && !icsk->icsk_bind_hash); 2640 2641 if (sk->sk_frag.page) { 2642 put_page(sk->sk_frag.page); 2643 sk->sk_frag.page = NULL; 2644 sk->sk_frag.offset = 0; 2645 } 2646 2647 sk->sk_error_report(sk); 2648 return 0; 2649 } 2650 EXPORT_SYMBOL(tcp_disconnect); 2651 2652 static inline bool tcp_can_repair_sock(const struct sock *sk) 2653 { 2654 return ns_capable(sock_net(sk)->user_ns, CAP_NET_ADMIN) && 2655 (sk->sk_state != TCP_LISTEN); 2656 } 2657 2658 static int tcp_repair_set_window(struct tcp_sock *tp, char __user *optbuf, int len) 2659 { 2660 struct tcp_repair_window opt; 2661 2662 if (!tp->repair) 2663 return -EPERM; 2664 2665 if (len != sizeof(opt)) 2666 return -EINVAL; 2667 2668 if (copy_from_user(&opt, optbuf, sizeof(opt))) 2669 return -EFAULT; 2670 2671 if (opt.max_window < opt.snd_wnd) 2672 return -EINVAL; 2673 2674 if (after(opt.snd_wl1, tp->rcv_nxt + opt.rcv_wnd)) 2675 return -EINVAL; 2676 2677 if (after(opt.rcv_wup, tp->rcv_nxt)) 2678 return -EINVAL; 2679 2680 tp->snd_wl1 = opt.snd_wl1; 2681 tp->snd_wnd = opt.snd_wnd; 2682 tp->max_window = opt.max_window; 2683 2684 tp->rcv_wnd = opt.rcv_wnd; 2685 tp->rcv_wup = opt.rcv_wup; 2686 2687 return 0; 2688 } 2689 2690 static int tcp_repair_options_est(struct sock *sk, 2691 struct tcp_repair_opt __user *optbuf, unsigned int len) 2692 { 2693 struct tcp_sock *tp = tcp_sk(sk); 2694 struct tcp_repair_opt opt; 2695 2696 while (len >= sizeof(opt)) { 2697 if (copy_from_user(&opt, optbuf, sizeof(opt))) 2698 return -EFAULT; 2699 2700 optbuf++; 2701 len -= sizeof(opt); 2702 2703 switch (opt.opt_code) { 2704 case TCPOPT_MSS: 2705 tp->rx_opt.mss_clamp = opt.opt_val; 2706 tcp_mtup_init(sk); 2707 break; 2708 case TCPOPT_WINDOW: 2709 { 2710 u16 snd_wscale = opt.opt_val & 0xFFFF; 2711 u16 rcv_wscale = opt.opt_val >> 16; 2712 2713 if (snd_wscale > TCP_MAX_WSCALE || rcv_wscale > TCP_MAX_WSCALE) 2714 return -EFBIG; 2715 2716 tp->rx_opt.snd_wscale = snd_wscale; 2717 tp->rx_opt.rcv_wscale = rcv_wscale; 2718 tp->rx_opt.wscale_ok = 1; 2719 } 2720 break; 2721 case TCPOPT_SACK_PERM: 2722 if (opt.opt_val != 0) 2723 return -EINVAL; 2724 2725 tp->rx_opt.sack_ok |= TCP_SACK_SEEN; 2726 break; 2727 case TCPOPT_TIMESTAMP: 2728 if (opt.opt_val != 0) 2729 return -EINVAL; 2730 2731 tp->rx_opt.tstamp_ok = 1; 2732 break; 2733 } 2734 } 2735 2736 return 0; 2737 } 2738 2739 /* 2740 * Socket option code for TCP. 2741 */ 2742 static int do_tcp_setsockopt(struct sock *sk, int level, 2743 int optname, char __user *optval, unsigned int optlen) 2744 { 2745 struct tcp_sock *tp = tcp_sk(sk); 2746 struct inet_connection_sock *icsk = inet_csk(sk); 2747 struct net *net = sock_net(sk); 2748 int val; 2749 int err = 0; 2750 2751 /* These are data/string values, all the others are ints */ 2752 switch (optname) { 2753 case TCP_CONGESTION: { 2754 char name[TCP_CA_NAME_MAX]; 2755 2756 if (optlen < 1) 2757 return -EINVAL; 2758 2759 val = strncpy_from_user(name, optval, 2760 min_t(long, TCP_CA_NAME_MAX-1, optlen)); 2761 if (val < 0) 2762 return -EFAULT; 2763 name[val] = 0; 2764 2765 lock_sock(sk); 2766 err = tcp_set_congestion_control(sk, name, true, true); 2767 release_sock(sk); 2768 return err; 2769 } 2770 case TCP_ULP: { 2771 char name[TCP_ULP_NAME_MAX]; 2772 2773 if (optlen < 1) 2774 return -EINVAL; 2775 2776 val = strncpy_from_user(name, optval, 2777 min_t(long, TCP_ULP_NAME_MAX - 1, 2778 optlen)); 2779 if (val < 0) 2780 return -EFAULT; 2781 name[val] = 0; 2782 2783 lock_sock(sk); 2784 err = tcp_set_ulp(sk, name); 2785 release_sock(sk); 2786 return err; 2787 } 2788 case TCP_FASTOPEN_KEY: { 2789 __u8 key[TCP_FASTOPEN_KEY_LENGTH]; 2790 2791 if (optlen != sizeof(key)) 2792 return -EINVAL; 2793 2794 if (copy_from_user(key, optval, optlen)) 2795 return -EFAULT; 2796 2797 return tcp_fastopen_reset_cipher(net, sk, key, sizeof(key)); 2798 } 2799 default: 2800 /* fallthru */ 2801 break; 2802 } 2803 2804 if (optlen < sizeof(int)) 2805 return -EINVAL; 2806 2807 if (get_user(val, (int __user *)optval)) 2808 return -EFAULT; 2809 2810 lock_sock(sk); 2811 2812 switch (optname) { 2813 case TCP_MAXSEG: 2814 /* Values greater than interface MTU won't take effect. However 2815 * at the point when this call is done we typically don't yet 2816 * know which interface is going to be used 2817 */ 2818 if (val && (val < TCP_MIN_MSS || val > MAX_TCP_WINDOW)) { 2819 err = -EINVAL; 2820 break; 2821 } 2822 tp->rx_opt.user_mss = val; 2823 break; 2824 2825 case TCP_NODELAY: 2826 if (val) { 2827 /* TCP_NODELAY is weaker than TCP_CORK, so that 2828 * this option on corked socket is remembered, but 2829 * it is not activated until cork is cleared. 2830 * 2831 * However, when TCP_NODELAY is set we make 2832 * an explicit push, which overrides even TCP_CORK 2833 * for currently queued segments. 2834 */ 2835 tp->nonagle |= TCP_NAGLE_OFF|TCP_NAGLE_PUSH; 2836 tcp_push_pending_frames(sk); 2837 } else { 2838 tp->nonagle &= ~TCP_NAGLE_OFF; 2839 } 2840 break; 2841 2842 case TCP_THIN_LINEAR_TIMEOUTS: 2843 if (val < 0 || val > 1) 2844 err = -EINVAL; 2845 else 2846 tp->thin_lto = val; 2847 break; 2848 2849 case TCP_THIN_DUPACK: 2850 if (val < 0 || val > 1) 2851 err = -EINVAL; 2852 break; 2853 2854 case TCP_REPAIR: 2855 if (!tcp_can_repair_sock(sk)) 2856 err = -EPERM; 2857 else if (val == TCP_REPAIR_ON) { 2858 tp->repair = 1; 2859 sk->sk_reuse = SK_FORCE_REUSE; 2860 tp->repair_queue = TCP_NO_QUEUE; 2861 } else if (val == TCP_REPAIR_OFF) { 2862 tp->repair = 0; 2863 sk->sk_reuse = SK_NO_REUSE; 2864 tcp_send_window_probe(sk); 2865 } else if (val == TCP_REPAIR_OFF_NO_WP) { 2866 tp->repair = 0; 2867 sk->sk_reuse = SK_NO_REUSE; 2868 } else 2869 err = -EINVAL; 2870 2871 break; 2872 2873 case TCP_REPAIR_QUEUE: 2874 if (!tp->repair) 2875 err = -EPERM; 2876 else if ((unsigned int)val < TCP_QUEUES_NR) 2877 tp->repair_queue = val; 2878 else 2879 err = -EINVAL; 2880 break; 2881 2882 case TCP_QUEUE_SEQ: 2883 if (sk->sk_state != TCP_CLOSE) 2884 err = -EPERM; 2885 else if (tp->repair_queue == TCP_SEND_QUEUE) 2886 tp->write_seq = val; 2887 else if (tp->repair_queue == TCP_RECV_QUEUE) 2888 tp->rcv_nxt = val; 2889 else 2890 err = -EINVAL; 2891 break; 2892 2893 case TCP_REPAIR_OPTIONS: 2894 if (!tp->repair) 2895 err = -EINVAL; 2896 else if (sk->sk_state == TCP_ESTABLISHED) 2897 err = tcp_repair_options_est(sk, 2898 (struct tcp_repair_opt __user *)optval, 2899 optlen); 2900 else 2901 err = -EPERM; 2902 break; 2903 2904 case TCP_CORK: 2905 /* When set indicates to always queue non-full frames. 2906 * Later the user clears this option and we transmit 2907 * any pending partial frames in the queue. This is 2908 * meant to be used alongside sendfile() to get properly 2909 * filled frames when the user (for example) must write 2910 * out headers with a write() call first and then use 2911 * sendfile to send out the data parts. 2912 * 2913 * TCP_CORK can be set together with TCP_NODELAY and it is 2914 * stronger than TCP_NODELAY. 2915 */ 2916 if (val) { 2917 tp->nonagle |= TCP_NAGLE_CORK; 2918 } else { 2919 tp->nonagle &= ~TCP_NAGLE_CORK; 2920 if (tp->nonagle&TCP_NAGLE_OFF) 2921 tp->nonagle |= TCP_NAGLE_PUSH; 2922 tcp_push_pending_frames(sk); 2923 } 2924 break; 2925 2926 case TCP_KEEPIDLE: 2927 if (val < 1 || val > MAX_TCP_KEEPIDLE) 2928 err = -EINVAL; 2929 else { 2930 tp->keepalive_time = val * HZ; 2931 if (sock_flag(sk, SOCK_KEEPOPEN) && 2932 !((1 << sk->sk_state) & 2933 (TCPF_CLOSE | TCPF_LISTEN))) { 2934 u32 elapsed = keepalive_time_elapsed(tp); 2935 if (tp->keepalive_time > elapsed) 2936 elapsed = tp->keepalive_time - elapsed; 2937 else 2938 elapsed = 0; 2939 inet_csk_reset_keepalive_timer(sk, elapsed); 2940 } 2941 } 2942 break; 2943 case TCP_KEEPINTVL: 2944 if (val < 1 || val > MAX_TCP_KEEPINTVL) 2945 err = -EINVAL; 2946 else 2947 tp->keepalive_intvl = val * HZ; 2948 break; 2949 case TCP_KEEPCNT: 2950 if (val < 1 || val > MAX_TCP_KEEPCNT) 2951 err = -EINVAL; 2952 else 2953 tp->keepalive_probes = val; 2954 break; 2955 case TCP_SYNCNT: 2956 if (val < 1 || val > MAX_TCP_SYNCNT) 2957 err = -EINVAL; 2958 else 2959 icsk->icsk_syn_retries = val; 2960 break; 2961 2962 case TCP_SAVE_SYN: 2963 if (val < 0 || val > 1) 2964 err = -EINVAL; 2965 else 2966 tp->save_syn = val; 2967 break; 2968 2969 case TCP_LINGER2: 2970 if (val < 0) 2971 tp->linger2 = -1; 2972 else if (val > net->ipv4.sysctl_tcp_fin_timeout / HZ) 2973 tp->linger2 = 0; 2974 else 2975 tp->linger2 = val * HZ; 2976 break; 2977 2978 case TCP_DEFER_ACCEPT: 2979 /* Translate value in seconds to number of retransmits */ 2980 icsk->icsk_accept_queue.rskq_defer_accept = 2981 secs_to_retrans(val, TCP_TIMEOUT_INIT / HZ, 2982 TCP_RTO_MAX / HZ); 2983 break; 2984 2985 case TCP_WINDOW_CLAMP: 2986 if (!val) { 2987 if (sk->sk_state != TCP_CLOSE) { 2988 err = -EINVAL; 2989 break; 2990 } 2991 tp->window_clamp = 0; 2992 } else 2993 tp->window_clamp = val < SOCK_MIN_RCVBUF / 2 ? 2994 SOCK_MIN_RCVBUF / 2 : val; 2995 break; 2996 2997 case TCP_QUICKACK: 2998 if (!val) { 2999 inet_csk_enter_pingpong_mode(sk); 3000 } else { 3001 inet_csk_exit_pingpong_mode(sk); 3002 if ((1 << sk->sk_state) & 3003 (TCPF_ESTABLISHED | TCPF_CLOSE_WAIT) && 3004 inet_csk_ack_scheduled(sk)) { 3005 icsk->icsk_ack.pending |= ICSK_ACK_PUSHED; 3006 tcp_cleanup_rbuf(sk, 1); 3007 if (!(val & 1)) 3008 inet_csk_enter_pingpong_mode(sk); 3009 } 3010 } 3011 break; 3012 3013 #ifdef CONFIG_TCP_MD5SIG 3014 case TCP_MD5SIG: 3015 case TCP_MD5SIG_EXT: 3016 if ((1 << sk->sk_state) & (TCPF_CLOSE | TCPF_LISTEN)) 3017 err = tp->af_specific->md5_parse(sk, optname, optval, optlen); 3018 else 3019 err = -EINVAL; 3020 break; 3021 #endif 3022 case TCP_USER_TIMEOUT: 3023 /* Cap the max time in ms TCP will retry or probe the window 3024 * before giving up and aborting (ETIMEDOUT) a connection. 3025 */ 3026 if (val < 0) 3027 err = -EINVAL; 3028 else 3029 icsk->icsk_user_timeout = val; 3030 break; 3031 3032 case TCP_FASTOPEN: 3033 if (val >= 0 && ((1 << sk->sk_state) & (TCPF_CLOSE | 3034 TCPF_LISTEN))) { 3035 tcp_fastopen_init_key_once(net); 3036 3037 fastopen_queue_tune(sk, val); 3038 } else { 3039 err = -EINVAL; 3040 } 3041 break; 3042 case TCP_FASTOPEN_CONNECT: 3043 if (val > 1 || val < 0) { 3044 err = -EINVAL; 3045 } else if (net->ipv4.sysctl_tcp_fastopen & TFO_CLIENT_ENABLE) { 3046 if (sk->sk_state == TCP_CLOSE) 3047 tp->fastopen_connect = val; 3048 else 3049 err = -EINVAL; 3050 } else { 3051 err = -EOPNOTSUPP; 3052 } 3053 break; 3054 case TCP_FASTOPEN_NO_COOKIE: 3055 if (val > 1 || val < 0) 3056 err = -EINVAL; 3057 else if (!((1 << sk->sk_state) & (TCPF_CLOSE | TCPF_LISTEN))) 3058 err = -EINVAL; 3059 else 3060 tp->fastopen_no_cookie = val; 3061 break; 3062 case TCP_TIMESTAMP: 3063 if (!tp->repair) 3064 err = -EPERM; 3065 else 3066 tp->tsoffset = val - tcp_time_stamp_raw(); 3067 break; 3068 case TCP_REPAIR_WINDOW: 3069 err = tcp_repair_set_window(tp, optval, optlen); 3070 break; 3071 case TCP_NOTSENT_LOWAT: 3072 tp->notsent_lowat = val; 3073 sk->sk_write_space(sk); 3074 break; 3075 case TCP_INQ: 3076 if (val > 1 || val < 0) 3077 err = -EINVAL; 3078 else 3079 tp->recvmsg_inq = val; 3080 break; 3081 default: 3082 err = -ENOPROTOOPT; 3083 break; 3084 } 3085 3086 release_sock(sk); 3087 return err; 3088 } 3089 3090 int tcp_setsockopt(struct sock *sk, int level, int optname, char __user *optval, 3091 unsigned int optlen) 3092 { 3093 const struct inet_connection_sock *icsk = inet_csk(sk); 3094 3095 if (level != SOL_TCP) 3096 return icsk->icsk_af_ops->setsockopt(sk, level, optname, 3097 optval, optlen); 3098 return do_tcp_setsockopt(sk, level, optname, optval, optlen); 3099 } 3100 EXPORT_SYMBOL(tcp_setsockopt); 3101 3102 #ifdef CONFIG_COMPAT 3103 int compat_tcp_setsockopt(struct sock *sk, int level, int optname, 3104 char __user *optval, unsigned int optlen) 3105 { 3106 if (level != SOL_TCP) 3107 return inet_csk_compat_setsockopt(sk, level, optname, 3108 optval, optlen); 3109 return do_tcp_setsockopt(sk, level, optname, optval, optlen); 3110 } 3111 EXPORT_SYMBOL(compat_tcp_setsockopt); 3112 #endif 3113 3114 static void tcp_get_info_chrono_stats(const struct tcp_sock *tp, 3115 struct tcp_info *info) 3116 { 3117 u64 stats[__TCP_CHRONO_MAX], total = 0; 3118 enum tcp_chrono i; 3119 3120 for (i = TCP_CHRONO_BUSY; i < __TCP_CHRONO_MAX; ++i) { 3121 stats[i] = tp->chrono_stat[i - 1]; 3122 if (i == tp->chrono_type) 3123 stats[i] += tcp_jiffies32 - tp->chrono_start; 3124 stats[i] *= USEC_PER_SEC / HZ; 3125 total += stats[i]; 3126 } 3127 3128 info->tcpi_busy_time = total; 3129 info->tcpi_rwnd_limited = stats[TCP_CHRONO_RWND_LIMITED]; 3130 info->tcpi_sndbuf_limited = stats[TCP_CHRONO_SNDBUF_LIMITED]; 3131 } 3132 3133 /* Return information about state of tcp endpoint in API format. */ 3134 void tcp_get_info(struct sock *sk, struct tcp_info *info) 3135 { 3136 const struct tcp_sock *tp = tcp_sk(sk); /* iff sk_type == SOCK_STREAM */ 3137 const struct inet_connection_sock *icsk = inet_csk(sk); 3138 unsigned long rate; 3139 u32 now; 3140 u64 rate64; 3141 bool slow; 3142 3143 memset(info, 0, sizeof(*info)); 3144 if (sk->sk_type != SOCK_STREAM) 3145 return; 3146 3147 info->tcpi_state = inet_sk_state_load(sk); 3148 3149 /* Report meaningful fields for all TCP states, including listeners */ 3150 rate = READ_ONCE(sk->sk_pacing_rate); 3151 rate64 = (rate != ~0UL) ? rate : ~0ULL; 3152 info->tcpi_pacing_rate = rate64; 3153 3154 rate = READ_ONCE(sk->sk_max_pacing_rate); 3155 rate64 = (rate != ~0UL) ? rate : ~0ULL; 3156 info->tcpi_max_pacing_rate = rate64; 3157 3158 info->tcpi_reordering = tp->reordering; 3159 info->tcpi_snd_cwnd = tp->snd_cwnd; 3160 3161 if (info->tcpi_state == TCP_LISTEN) { 3162 /* listeners aliased fields : 3163 * tcpi_unacked -> Number of children ready for accept() 3164 * tcpi_sacked -> max backlog 3165 */ 3166 info->tcpi_unacked = sk->sk_ack_backlog; 3167 info->tcpi_sacked = sk->sk_max_ack_backlog; 3168 return; 3169 } 3170 3171 slow = lock_sock_fast(sk); 3172 3173 info->tcpi_ca_state = icsk->icsk_ca_state; 3174 info->tcpi_retransmits = icsk->icsk_retransmits; 3175 info->tcpi_probes = icsk->icsk_probes_out; 3176 info->tcpi_backoff = icsk->icsk_backoff; 3177 3178 if (tp->rx_opt.tstamp_ok) 3179 info->tcpi_options |= TCPI_OPT_TIMESTAMPS; 3180 if (tcp_is_sack(tp)) 3181 info->tcpi_options |= TCPI_OPT_SACK; 3182 if (tp->rx_opt.wscale_ok) { 3183 info->tcpi_options |= TCPI_OPT_WSCALE; 3184 info->tcpi_snd_wscale = tp->rx_opt.snd_wscale; 3185 info->tcpi_rcv_wscale = tp->rx_opt.rcv_wscale; 3186 } 3187 3188 if (tp->ecn_flags & TCP_ECN_OK) 3189 info->tcpi_options |= TCPI_OPT_ECN; 3190 if (tp->ecn_flags & TCP_ECN_SEEN) 3191 info->tcpi_options |= TCPI_OPT_ECN_SEEN; 3192 if (tp->syn_data_acked) 3193 info->tcpi_options |= TCPI_OPT_SYN_DATA; 3194 3195 info->tcpi_rto = jiffies_to_usecs(icsk->icsk_rto); 3196 info->tcpi_ato = jiffies_to_usecs(icsk->icsk_ack.ato); 3197 info->tcpi_snd_mss = tp->mss_cache; 3198 info->tcpi_rcv_mss = icsk->icsk_ack.rcv_mss; 3199 3200 info->tcpi_unacked = tp->packets_out; 3201 info->tcpi_sacked = tp->sacked_out; 3202 3203 info->tcpi_lost = tp->lost_out; 3204 info->tcpi_retrans = tp->retrans_out; 3205 3206 now = tcp_jiffies32; 3207 info->tcpi_last_data_sent = jiffies_to_msecs(now - tp->lsndtime); 3208 info->tcpi_last_data_recv = jiffies_to_msecs(now - icsk->icsk_ack.lrcvtime); 3209 info->tcpi_last_ack_recv = jiffies_to_msecs(now - tp->rcv_tstamp); 3210 3211 info->tcpi_pmtu = icsk->icsk_pmtu_cookie; 3212 info->tcpi_rcv_ssthresh = tp->rcv_ssthresh; 3213 info->tcpi_rtt = tp->srtt_us >> 3; 3214 info->tcpi_rttvar = tp->mdev_us >> 2; 3215 info->tcpi_snd_ssthresh = tp->snd_ssthresh; 3216 info->tcpi_advmss = tp->advmss; 3217 3218 info->tcpi_rcv_rtt = tp->rcv_rtt_est.rtt_us >> 3; 3219 info->tcpi_rcv_space = tp->rcvq_space.space; 3220 3221 info->tcpi_total_retrans = tp->total_retrans; 3222 3223 info->tcpi_bytes_acked = tp->bytes_acked; 3224 info->tcpi_bytes_received = tp->bytes_received; 3225 info->tcpi_notsent_bytes = max_t(int, 0, tp->write_seq - tp->snd_nxt); 3226 tcp_get_info_chrono_stats(tp, info); 3227 3228 info->tcpi_segs_out = tp->segs_out; 3229 info->tcpi_segs_in = tp->segs_in; 3230 3231 info->tcpi_min_rtt = tcp_min_rtt(tp); 3232 info->tcpi_data_segs_in = tp->data_segs_in; 3233 info->tcpi_data_segs_out = tp->data_segs_out; 3234 3235 info->tcpi_delivery_rate_app_limited = tp->rate_app_limited ? 1 : 0; 3236 rate64 = tcp_compute_delivery_rate(tp); 3237 if (rate64) 3238 info->tcpi_delivery_rate = rate64; 3239 info->tcpi_delivered = tp->delivered; 3240 info->tcpi_delivered_ce = tp->delivered_ce; 3241 info->tcpi_bytes_sent = tp->bytes_sent; 3242 info->tcpi_bytes_retrans = tp->bytes_retrans; 3243 info->tcpi_dsack_dups = tp->dsack_dups; 3244 info->tcpi_reord_seen = tp->reord_seen; 3245 unlock_sock_fast(sk, slow); 3246 } 3247 EXPORT_SYMBOL_GPL(tcp_get_info); 3248 3249 static size_t tcp_opt_stats_get_size(void) 3250 { 3251 return 3252 nla_total_size_64bit(sizeof(u64)) + /* TCP_NLA_BUSY */ 3253 nla_total_size_64bit(sizeof(u64)) + /* TCP_NLA_RWND_LIMITED */ 3254 nla_total_size_64bit(sizeof(u64)) + /* TCP_NLA_SNDBUF_LIMITED */ 3255 nla_total_size_64bit(sizeof(u64)) + /* TCP_NLA_DATA_SEGS_OUT */ 3256 nla_total_size_64bit(sizeof(u64)) + /* TCP_NLA_TOTAL_RETRANS */ 3257 nla_total_size_64bit(sizeof(u64)) + /* TCP_NLA_PACING_RATE */ 3258 nla_total_size_64bit(sizeof(u64)) + /* TCP_NLA_DELIVERY_RATE */ 3259 nla_total_size(sizeof(u32)) + /* TCP_NLA_SND_CWND */ 3260 nla_total_size(sizeof(u32)) + /* TCP_NLA_REORDERING */ 3261 nla_total_size(sizeof(u32)) + /* TCP_NLA_MIN_RTT */ 3262 nla_total_size(sizeof(u8)) + /* TCP_NLA_RECUR_RETRANS */ 3263 nla_total_size(sizeof(u8)) + /* TCP_NLA_DELIVERY_RATE_APP_LMT */ 3264 nla_total_size(sizeof(u32)) + /* TCP_NLA_SNDQ_SIZE */ 3265 nla_total_size(sizeof(u8)) + /* TCP_NLA_CA_STATE */ 3266 nla_total_size(sizeof(u32)) + /* TCP_NLA_SND_SSTHRESH */ 3267 nla_total_size(sizeof(u32)) + /* TCP_NLA_DELIVERED */ 3268 nla_total_size(sizeof(u32)) + /* TCP_NLA_DELIVERED_CE */ 3269 nla_total_size_64bit(sizeof(u64)) + /* TCP_NLA_BYTES_SENT */ 3270 nla_total_size_64bit(sizeof(u64)) + /* TCP_NLA_BYTES_RETRANS */ 3271 nla_total_size(sizeof(u32)) + /* TCP_NLA_DSACK_DUPS */ 3272 nla_total_size(sizeof(u32)) + /* TCP_NLA_REORD_SEEN */ 3273 nla_total_size(sizeof(u32)) + /* TCP_NLA_SRTT */ 3274 0; 3275 } 3276 3277 struct sk_buff *tcp_get_timestamping_opt_stats(const struct sock *sk) 3278 { 3279 const struct tcp_sock *tp = tcp_sk(sk); 3280 struct sk_buff *stats; 3281 struct tcp_info info; 3282 unsigned long rate; 3283 u64 rate64; 3284 3285 stats = alloc_skb(tcp_opt_stats_get_size(), GFP_ATOMIC); 3286 if (!stats) 3287 return NULL; 3288 3289 tcp_get_info_chrono_stats(tp, &info); 3290 nla_put_u64_64bit(stats, TCP_NLA_BUSY, 3291 info.tcpi_busy_time, TCP_NLA_PAD); 3292 nla_put_u64_64bit(stats, TCP_NLA_RWND_LIMITED, 3293 info.tcpi_rwnd_limited, TCP_NLA_PAD); 3294 nla_put_u64_64bit(stats, TCP_NLA_SNDBUF_LIMITED, 3295 info.tcpi_sndbuf_limited, TCP_NLA_PAD); 3296 nla_put_u64_64bit(stats, TCP_NLA_DATA_SEGS_OUT, 3297 tp->data_segs_out, TCP_NLA_PAD); 3298 nla_put_u64_64bit(stats, TCP_NLA_TOTAL_RETRANS, 3299 tp->total_retrans, TCP_NLA_PAD); 3300 3301 rate = READ_ONCE(sk->sk_pacing_rate); 3302 rate64 = (rate != ~0UL) ? rate : ~0ULL; 3303 nla_put_u64_64bit(stats, TCP_NLA_PACING_RATE, rate64, TCP_NLA_PAD); 3304 3305 rate64 = tcp_compute_delivery_rate(tp); 3306 nla_put_u64_64bit(stats, TCP_NLA_DELIVERY_RATE, rate64, TCP_NLA_PAD); 3307 3308 nla_put_u32(stats, TCP_NLA_SND_CWND, tp->snd_cwnd); 3309 nla_put_u32(stats, TCP_NLA_REORDERING, tp->reordering); 3310 nla_put_u32(stats, TCP_NLA_MIN_RTT, tcp_min_rtt(tp)); 3311 3312 nla_put_u8(stats, TCP_NLA_RECUR_RETRANS, inet_csk(sk)->icsk_retransmits); 3313 nla_put_u8(stats, TCP_NLA_DELIVERY_RATE_APP_LMT, !!tp->rate_app_limited); 3314 nla_put_u32(stats, TCP_NLA_SND_SSTHRESH, tp->snd_ssthresh); 3315 nla_put_u32(stats, TCP_NLA_DELIVERED, tp->delivered); 3316 nla_put_u32(stats, TCP_NLA_DELIVERED_CE, tp->delivered_ce); 3317 3318 nla_put_u32(stats, TCP_NLA_SNDQ_SIZE, tp->write_seq - tp->snd_una); 3319 nla_put_u8(stats, TCP_NLA_CA_STATE, inet_csk(sk)->icsk_ca_state); 3320 3321 nla_put_u64_64bit(stats, TCP_NLA_BYTES_SENT, tp->bytes_sent, 3322 TCP_NLA_PAD); 3323 nla_put_u64_64bit(stats, TCP_NLA_BYTES_RETRANS, tp->bytes_retrans, 3324 TCP_NLA_PAD); 3325 nla_put_u32(stats, TCP_NLA_DSACK_DUPS, tp->dsack_dups); 3326 nla_put_u32(stats, TCP_NLA_REORD_SEEN, tp->reord_seen); 3327 nla_put_u32(stats, TCP_NLA_SRTT, tp->srtt_us >> 3); 3328 3329 return stats; 3330 } 3331 3332 static int do_tcp_getsockopt(struct sock *sk, int level, 3333 int optname, char __user *optval, int __user *optlen) 3334 { 3335 struct inet_connection_sock *icsk = inet_csk(sk); 3336 struct tcp_sock *tp = tcp_sk(sk); 3337 struct net *net = sock_net(sk); 3338 int val, len; 3339 3340 if (get_user(len, optlen)) 3341 return -EFAULT; 3342 3343 len = min_t(unsigned int, len, sizeof(int)); 3344 3345 if (len < 0) 3346 return -EINVAL; 3347 3348 switch (optname) { 3349 case TCP_MAXSEG: 3350 val = tp->mss_cache; 3351 if (!val && ((1 << sk->sk_state) & (TCPF_CLOSE | TCPF_LISTEN))) 3352 val = tp->rx_opt.user_mss; 3353 if (tp->repair) 3354 val = tp->rx_opt.mss_clamp; 3355 break; 3356 case TCP_NODELAY: 3357 val = !!(tp->nonagle&TCP_NAGLE_OFF); 3358 break; 3359 case TCP_CORK: 3360 val = !!(tp->nonagle&TCP_NAGLE_CORK); 3361 break; 3362 case TCP_KEEPIDLE: 3363 val = keepalive_time_when(tp) / HZ; 3364 break; 3365 case TCP_KEEPINTVL: 3366 val = keepalive_intvl_when(tp) / HZ; 3367 break; 3368 case TCP_KEEPCNT: 3369 val = keepalive_probes(tp); 3370 break; 3371 case TCP_SYNCNT: 3372 val = icsk->icsk_syn_retries ? : net->ipv4.sysctl_tcp_syn_retries; 3373 break; 3374 case TCP_LINGER2: 3375 val = tp->linger2; 3376 if (val >= 0) 3377 val = (val ? : net->ipv4.sysctl_tcp_fin_timeout) / HZ; 3378 break; 3379 case TCP_DEFER_ACCEPT: 3380 val = retrans_to_secs(icsk->icsk_accept_queue.rskq_defer_accept, 3381 TCP_TIMEOUT_INIT / HZ, TCP_RTO_MAX / HZ); 3382 break; 3383 case TCP_WINDOW_CLAMP: 3384 val = tp->window_clamp; 3385 break; 3386 case TCP_INFO: { 3387 struct tcp_info info; 3388 3389 if (get_user(len, optlen)) 3390 return -EFAULT; 3391 3392 tcp_get_info(sk, &info); 3393 3394 len = min_t(unsigned int, len, sizeof(info)); 3395 if (put_user(len, optlen)) 3396 return -EFAULT; 3397 if (copy_to_user(optval, &info, len)) 3398 return -EFAULT; 3399 return 0; 3400 } 3401 case TCP_CC_INFO: { 3402 const struct tcp_congestion_ops *ca_ops; 3403 union tcp_cc_info info; 3404 size_t sz = 0; 3405 int attr; 3406 3407 if (get_user(len, optlen)) 3408 return -EFAULT; 3409 3410 ca_ops = icsk->icsk_ca_ops; 3411 if (ca_ops && ca_ops->get_info) 3412 sz = ca_ops->get_info(sk, ~0U, &attr, &info); 3413 3414 len = min_t(unsigned int, len, sz); 3415 if (put_user(len, optlen)) 3416 return -EFAULT; 3417 if (copy_to_user(optval, &info, len)) 3418 return -EFAULT; 3419 return 0; 3420 } 3421 case TCP_QUICKACK: 3422 val = !inet_csk_in_pingpong_mode(sk); 3423 break; 3424 3425 case TCP_CONGESTION: 3426 if (get_user(len, optlen)) 3427 return -EFAULT; 3428 len = min_t(unsigned int, len, TCP_CA_NAME_MAX); 3429 if (put_user(len, optlen)) 3430 return -EFAULT; 3431 if (copy_to_user(optval, icsk->icsk_ca_ops->name, len)) 3432 return -EFAULT; 3433 return 0; 3434 3435 case TCP_ULP: 3436 if (get_user(len, optlen)) 3437 return -EFAULT; 3438 len = min_t(unsigned int, len, TCP_ULP_NAME_MAX); 3439 if (!icsk->icsk_ulp_ops) { 3440 if (put_user(0, optlen)) 3441 return -EFAULT; 3442 return 0; 3443 } 3444 if (put_user(len, optlen)) 3445 return -EFAULT; 3446 if (copy_to_user(optval, icsk->icsk_ulp_ops->name, len)) 3447 return -EFAULT; 3448 return 0; 3449 3450 case TCP_FASTOPEN_KEY: { 3451 __u8 key[TCP_FASTOPEN_KEY_LENGTH]; 3452 struct tcp_fastopen_context *ctx; 3453 3454 if (get_user(len, optlen)) 3455 return -EFAULT; 3456 3457 rcu_read_lock(); 3458 ctx = rcu_dereference(icsk->icsk_accept_queue.fastopenq.ctx); 3459 if (ctx) 3460 memcpy(key, ctx->key, sizeof(key)); 3461 else 3462 len = 0; 3463 rcu_read_unlock(); 3464 3465 len = min_t(unsigned int, len, sizeof(key)); 3466 if (put_user(len, optlen)) 3467 return -EFAULT; 3468 if (copy_to_user(optval, key, len)) 3469 return -EFAULT; 3470 return 0; 3471 } 3472 case TCP_THIN_LINEAR_TIMEOUTS: 3473 val = tp->thin_lto; 3474 break; 3475 3476 case TCP_THIN_DUPACK: 3477 val = 0; 3478 break; 3479 3480 case TCP_REPAIR: 3481 val = tp->repair; 3482 break; 3483 3484 case TCP_REPAIR_QUEUE: 3485 if (tp->repair) 3486 val = tp->repair_queue; 3487 else 3488 return -EINVAL; 3489 break; 3490 3491 case TCP_REPAIR_WINDOW: { 3492 struct tcp_repair_window opt; 3493 3494 if (get_user(len, optlen)) 3495 return -EFAULT; 3496 3497 if (len != sizeof(opt)) 3498 return -EINVAL; 3499 3500 if (!tp->repair) 3501 return -EPERM; 3502 3503 opt.snd_wl1 = tp->snd_wl1; 3504 opt.snd_wnd = tp->snd_wnd; 3505 opt.max_window = tp->max_window; 3506 opt.rcv_wnd = tp->rcv_wnd; 3507 opt.rcv_wup = tp->rcv_wup; 3508 3509 if (copy_to_user(optval, &opt, len)) 3510 return -EFAULT; 3511 return 0; 3512 } 3513 case TCP_QUEUE_SEQ: 3514 if (tp->repair_queue == TCP_SEND_QUEUE) 3515 val = tp->write_seq; 3516 else if (tp->repair_queue == TCP_RECV_QUEUE) 3517 val = tp->rcv_nxt; 3518 else 3519 return -EINVAL; 3520 break; 3521 3522 case TCP_USER_TIMEOUT: 3523 val = icsk->icsk_user_timeout; 3524 break; 3525 3526 case TCP_FASTOPEN: 3527 val = icsk->icsk_accept_queue.fastopenq.max_qlen; 3528 break; 3529 3530 case TCP_FASTOPEN_CONNECT: 3531 val = tp->fastopen_connect; 3532 break; 3533 3534 case TCP_FASTOPEN_NO_COOKIE: 3535 val = tp->fastopen_no_cookie; 3536 break; 3537 3538 case TCP_TIMESTAMP: 3539 val = tcp_time_stamp_raw() + tp->tsoffset; 3540 break; 3541 case TCP_NOTSENT_LOWAT: 3542 val = tp->notsent_lowat; 3543 break; 3544 case TCP_INQ: 3545 val = tp->recvmsg_inq; 3546 break; 3547 case TCP_SAVE_SYN: 3548 val = tp->save_syn; 3549 break; 3550 case TCP_SAVED_SYN: { 3551 if (get_user(len, optlen)) 3552 return -EFAULT; 3553 3554 lock_sock(sk); 3555 if (tp->saved_syn) { 3556 if (len < tp->saved_syn[0]) { 3557 if (put_user(tp->saved_syn[0], optlen)) { 3558 release_sock(sk); 3559 return -EFAULT; 3560 } 3561 release_sock(sk); 3562 return -EINVAL; 3563 } 3564 len = tp->saved_syn[0]; 3565 if (put_user(len, optlen)) { 3566 release_sock(sk); 3567 return -EFAULT; 3568 } 3569 if (copy_to_user(optval, tp->saved_syn + 1, len)) { 3570 release_sock(sk); 3571 return -EFAULT; 3572 } 3573 tcp_saved_syn_free(tp); 3574 release_sock(sk); 3575 } else { 3576 release_sock(sk); 3577 len = 0; 3578 if (put_user(len, optlen)) 3579 return -EFAULT; 3580 } 3581 return 0; 3582 } 3583 #ifdef CONFIG_MMU 3584 case TCP_ZEROCOPY_RECEIVE: { 3585 struct tcp_zerocopy_receive zc; 3586 int err; 3587 3588 if (get_user(len, optlen)) 3589 return -EFAULT; 3590 if (len != sizeof(zc)) 3591 return -EINVAL; 3592 if (copy_from_user(&zc, optval, len)) 3593 return -EFAULT; 3594 lock_sock(sk); 3595 err = tcp_zerocopy_receive(sk, &zc); 3596 release_sock(sk); 3597 if (!err && copy_to_user(optval, &zc, len)) 3598 err = -EFAULT; 3599 return err; 3600 } 3601 #endif 3602 default: 3603 return -ENOPROTOOPT; 3604 } 3605 3606 if (put_user(len, optlen)) 3607 return -EFAULT; 3608 if (copy_to_user(optval, &val, len)) 3609 return -EFAULT; 3610 return 0; 3611 } 3612 3613 int tcp_getsockopt(struct sock *sk, int level, int optname, char __user *optval, 3614 int __user *optlen) 3615 { 3616 struct inet_connection_sock *icsk = inet_csk(sk); 3617 3618 if (level != SOL_TCP) 3619 return icsk->icsk_af_ops->getsockopt(sk, level, optname, 3620 optval, optlen); 3621 return do_tcp_getsockopt(sk, level, optname, optval, optlen); 3622 } 3623 EXPORT_SYMBOL(tcp_getsockopt); 3624 3625 #ifdef CONFIG_COMPAT 3626 int compat_tcp_getsockopt(struct sock *sk, int level, int optname, 3627 char __user *optval, int __user *optlen) 3628 { 3629 if (level != SOL_TCP) 3630 return inet_csk_compat_getsockopt(sk, level, optname, 3631 optval, optlen); 3632 return do_tcp_getsockopt(sk, level, optname, optval, optlen); 3633 } 3634 EXPORT_SYMBOL(compat_tcp_getsockopt); 3635 #endif 3636 3637 #ifdef CONFIG_TCP_MD5SIG 3638 static DEFINE_PER_CPU(struct tcp_md5sig_pool, tcp_md5sig_pool); 3639 static DEFINE_MUTEX(tcp_md5sig_mutex); 3640 static bool tcp_md5sig_pool_populated = false; 3641 3642 static void __tcp_alloc_md5sig_pool(void) 3643 { 3644 struct crypto_ahash *hash; 3645 int cpu; 3646 3647 hash = crypto_alloc_ahash("md5", 0, CRYPTO_ALG_ASYNC); 3648 if (IS_ERR(hash)) 3649 return; 3650 3651 for_each_possible_cpu(cpu) { 3652 void *scratch = per_cpu(tcp_md5sig_pool, cpu).scratch; 3653 struct ahash_request *req; 3654 3655 if (!scratch) { 3656 scratch = kmalloc_node(sizeof(union tcp_md5sum_block) + 3657 sizeof(struct tcphdr), 3658 GFP_KERNEL, 3659 cpu_to_node(cpu)); 3660 if (!scratch) 3661 return; 3662 per_cpu(tcp_md5sig_pool, cpu).scratch = scratch; 3663 } 3664 if (per_cpu(tcp_md5sig_pool, cpu).md5_req) 3665 continue; 3666 3667 req = ahash_request_alloc(hash, GFP_KERNEL); 3668 if (!req) 3669 return; 3670 3671 ahash_request_set_callback(req, 0, NULL, NULL); 3672 3673 per_cpu(tcp_md5sig_pool, cpu).md5_req = req; 3674 } 3675 /* before setting tcp_md5sig_pool_populated, we must commit all writes 3676 * to memory. See smp_rmb() in tcp_get_md5sig_pool() 3677 */ 3678 smp_wmb(); 3679 tcp_md5sig_pool_populated = true; 3680 } 3681 3682 bool tcp_alloc_md5sig_pool(void) 3683 { 3684 if (unlikely(!tcp_md5sig_pool_populated)) { 3685 mutex_lock(&tcp_md5sig_mutex); 3686 3687 if (!tcp_md5sig_pool_populated) { 3688 __tcp_alloc_md5sig_pool(); 3689 if (tcp_md5sig_pool_populated) 3690 static_branch_inc(&tcp_md5_needed); 3691 } 3692 3693 mutex_unlock(&tcp_md5sig_mutex); 3694 } 3695 return tcp_md5sig_pool_populated; 3696 } 3697 EXPORT_SYMBOL(tcp_alloc_md5sig_pool); 3698 3699 3700 /** 3701 * tcp_get_md5sig_pool - get md5sig_pool for this user 3702 * 3703 * We use percpu structure, so if we succeed, we exit with preemption 3704 * and BH disabled, to make sure another thread or softirq handling 3705 * wont try to get same context. 3706 */ 3707 struct tcp_md5sig_pool *tcp_get_md5sig_pool(void) 3708 { 3709 local_bh_disable(); 3710 3711 if (tcp_md5sig_pool_populated) { 3712 /* coupled with smp_wmb() in __tcp_alloc_md5sig_pool() */ 3713 smp_rmb(); 3714 return this_cpu_ptr(&tcp_md5sig_pool); 3715 } 3716 local_bh_enable(); 3717 return NULL; 3718 } 3719 EXPORT_SYMBOL(tcp_get_md5sig_pool); 3720 3721 int tcp_md5_hash_skb_data(struct tcp_md5sig_pool *hp, 3722 const struct sk_buff *skb, unsigned int header_len) 3723 { 3724 struct scatterlist sg; 3725 const struct tcphdr *tp = tcp_hdr(skb); 3726 struct ahash_request *req = hp->md5_req; 3727 unsigned int i; 3728 const unsigned int head_data_len = skb_headlen(skb) > header_len ? 3729 skb_headlen(skb) - header_len : 0; 3730 const struct skb_shared_info *shi = skb_shinfo(skb); 3731 struct sk_buff *frag_iter; 3732 3733 sg_init_table(&sg, 1); 3734 3735 sg_set_buf(&sg, ((u8 *) tp) + header_len, head_data_len); 3736 ahash_request_set_crypt(req, &sg, NULL, head_data_len); 3737 if (crypto_ahash_update(req)) 3738 return 1; 3739 3740 for (i = 0; i < shi->nr_frags; ++i) { 3741 const struct skb_frag_struct *f = &shi->frags[i]; 3742 unsigned int offset = f->page_offset; 3743 struct page *page = skb_frag_page(f) + (offset >> PAGE_SHIFT); 3744 3745 sg_set_page(&sg, page, skb_frag_size(f), 3746 offset_in_page(offset)); 3747 ahash_request_set_crypt(req, &sg, NULL, skb_frag_size(f)); 3748 if (crypto_ahash_update(req)) 3749 return 1; 3750 } 3751 3752 skb_walk_frags(skb, frag_iter) 3753 if (tcp_md5_hash_skb_data(hp, frag_iter, 0)) 3754 return 1; 3755 3756 return 0; 3757 } 3758 EXPORT_SYMBOL(tcp_md5_hash_skb_data); 3759 3760 int tcp_md5_hash_key(struct tcp_md5sig_pool *hp, const struct tcp_md5sig_key *key) 3761 { 3762 struct scatterlist sg; 3763 3764 sg_init_one(&sg, key->key, key->keylen); 3765 ahash_request_set_crypt(hp->md5_req, &sg, NULL, key->keylen); 3766 return crypto_ahash_update(hp->md5_req); 3767 } 3768 EXPORT_SYMBOL(tcp_md5_hash_key); 3769 3770 #endif 3771 3772 void tcp_done(struct sock *sk) 3773 { 3774 struct request_sock *req = tcp_sk(sk)->fastopen_rsk; 3775 3776 if (sk->sk_state == TCP_SYN_SENT || sk->sk_state == TCP_SYN_RECV) 3777 TCP_INC_STATS(sock_net(sk), TCP_MIB_ATTEMPTFAILS); 3778 3779 tcp_set_state(sk, TCP_CLOSE); 3780 tcp_clear_xmit_timers(sk); 3781 if (req) 3782 reqsk_fastopen_remove(sk, req, false); 3783 3784 sk->sk_shutdown = SHUTDOWN_MASK; 3785 3786 if (!sock_flag(sk, SOCK_DEAD)) 3787 sk->sk_state_change(sk); 3788 else 3789 inet_csk_destroy_sock(sk); 3790 } 3791 EXPORT_SYMBOL_GPL(tcp_done); 3792 3793 int tcp_abort(struct sock *sk, int err) 3794 { 3795 if (!sk_fullsock(sk)) { 3796 if (sk->sk_state == TCP_NEW_SYN_RECV) { 3797 struct request_sock *req = inet_reqsk(sk); 3798 3799 local_bh_disable(); 3800 inet_csk_reqsk_queue_drop(req->rsk_listener, req); 3801 local_bh_enable(); 3802 return 0; 3803 } 3804 return -EOPNOTSUPP; 3805 } 3806 3807 /* Don't race with userspace socket closes such as tcp_close. */ 3808 lock_sock(sk); 3809 3810 if (sk->sk_state == TCP_LISTEN) { 3811 tcp_set_state(sk, TCP_CLOSE); 3812 inet_csk_listen_stop(sk); 3813 } 3814 3815 /* Don't race with BH socket closes such as inet_csk_listen_stop. */ 3816 local_bh_disable(); 3817 bh_lock_sock(sk); 3818 3819 if (!sock_flag(sk, SOCK_DEAD)) { 3820 sk->sk_err = err; 3821 /* This barrier is coupled with smp_rmb() in tcp_poll() */ 3822 smp_wmb(); 3823 sk->sk_error_report(sk); 3824 if (tcp_need_reset(sk->sk_state)) 3825 tcp_send_active_reset(sk, GFP_ATOMIC); 3826 tcp_done(sk); 3827 } 3828 3829 bh_unlock_sock(sk); 3830 local_bh_enable(); 3831 tcp_write_queue_purge(sk); 3832 release_sock(sk); 3833 return 0; 3834 } 3835 EXPORT_SYMBOL_GPL(tcp_abort); 3836 3837 extern struct tcp_congestion_ops tcp_reno; 3838 3839 static __initdata unsigned long thash_entries; 3840 static int __init set_thash_entries(char *str) 3841 { 3842 ssize_t ret; 3843 3844 if (!str) 3845 return 0; 3846 3847 ret = kstrtoul(str, 0, &thash_entries); 3848 if (ret) 3849 return 0; 3850 3851 return 1; 3852 } 3853 __setup("thash_entries=", set_thash_entries); 3854 3855 static void __init tcp_init_mem(void) 3856 { 3857 unsigned long limit = nr_free_buffer_pages() / 16; 3858 3859 limit = max(limit, 128UL); 3860 sysctl_tcp_mem[0] = limit / 4 * 3; /* 4.68 % */ 3861 sysctl_tcp_mem[1] = limit; /* 6.25 % */ 3862 sysctl_tcp_mem[2] = sysctl_tcp_mem[0] * 2; /* 9.37 % */ 3863 } 3864 3865 void __init tcp_init(void) 3866 { 3867 int max_rshare, max_wshare, cnt; 3868 unsigned long limit; 3869 unsigned int i; 3870 3871 BUILD_BUG_ON(sizeof(struct tcp_skb_cb) > 3872 FIELD_SIZEOF(struct sk_buff, cb)); 3873 3874 percpu_counter_init(&tcp_sockets_allocated, 0, GFP_KERNEL); 3875 percpu_counter_init(&tcp_orphan_count, 0, GFP_KERNEL); 3876 inet_hashinfo_init(&tcp_hashinfo); 3877 inet_hashinfo2_init(&tcp_hashinfo, "tcp_listen_portaddr_hash", 3878 thash_entries, 21, /* one slot per 2 MB*/ 3879 0, 64 * 1024); 3880 tcp_hashinfo.bind_bucket_cachep = 3881 kmem_cache_create("tcp_bind_bucket", 3882 sizeof(struct inet_bind_bucket), 0, 3883 SLAB_HWCACHE_ALIGN|SLAB_PANIC, NULL); 3884 3885 /* Size and allocate the main established and bind bucket 3886 * hash tables. 3887 * 3888 * The methodology is similar to that of the buffer cache. 3889 */ 3890 tcp_hashinfo.ehash = 3891 alloc_large_system_hash("TCP established", 3892 sizeof(struct inet_ehash_bucket), 3893 thash_entries, 3894 17, /* one slot per 128 KB of memory */ 3895 0, 3896 NULL, 3897 &tcp_hashinfo.ehash_mask, 3898 0, 3899 thash_entries ? 0 : 512 * 1024); 3900 for (i = 0; i <= tcp_hashinfo.ehash_mask; i++) 3901 INIT_HLIST_NULLS_HEAD(&tcp_hashinfo.ehash[i].chain, i); 3902 3903 if (inet_ehash_locks_alloc(&tcp_hashinfo)) 3904 panic("TCP: failed to alloc ehash_locks"); 3905 tcp_hashinfo.bhash = 3906 alloc_large_system_hash("TCP bind", 3907 sizeof(struct inet_bind_hashbucket), 3908 tcp_hashinfo.ehash_mask + 1, 3909 17, /* one slot per 128 KB of memory */ 3910 0, 3911 &tcp_hashinfo.bhash_size, 3912 NULL, 3913 0, 3914 64 * 1024); 3915 tcp_hashinfo.bhash_size = 1U << tcp_hashinfo.bhash_size; 3916 for (i = 0; i < tcp_hashinfo.bhash_size; i++) { 3917 spin_lock_init(&tcp_hashinfo.bhash[i].lock); 3918 INIT_HLIST_HEAD(&tcp_hashinfo.bhash[i].chain); 3919 } 3920 3921 3922 cnt = tcp_hashinfo.ehash_mask + 1; 3923 sysctl_tcp_max_orphans = cnt / 2; 3924 3925 tcp_init_mem(); 3926 /* Set per-socket limits to no more than 1/128 the pressure threshold */ 3927 limit = nr_free_buffer_pages() << (PAGE_SHIFT - 7); 3928 max_wshare = min(4UL*1024*1024, limit); 3929 max_rshare = min(6UL*1024*1024, limit); 3930 3931 init_net.ipv4.sysctl_tcp_wmem[0] = SK_MEM_QUANTUM; 3932 init_net.ipv4.sysctl_tcp_wmem[1] = 16*1024; 3933 init_net.ipv4.sysctl_tcp_wmem[2] = max(64*1024, max_wshare); 3934 3935 init_net.ipv4.sysctl_tcp_rmem[0] = SK_MEM_QUANTUM; 3936 init_net.ipv4.sysctl_tcp_rmem[1] = 131072; 3937 init_net.ipv4.sysctl_tcp_rmem[2] = max(131072, max_rshare); 3938 3939 pr_info("Hash tables configured (established %u bind %u)\n", 3940 tcp_hashinfo.ehash_mask + 1, tcp_hashinfo.bhash_size); 3941 3942 tcp_v4_init(); 3943 tcp_metrics_init(); 3944 BUG_ON(tcp_register_congestion_control(&tcp_reno) != 0); 3945 tcp_tasklet_init(); 3946 } 3947