1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * INET An implementation of the TCP/IP protocol suite for the LINUX 4 * operating system. INET is implemented using the BSD Socket 5 * interface as the means of communication with the user level. 6 * 7 * Implementation of the Transmission Control Protocol(TCP). 8 * 9 * Authors: Ross Biro 10 * Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG> 11 * Mark Evans, <evansmp@uhura.aston.ac.uk> 12 * Corey Minyard <wf-rch!minyard@relay.EU.net> 13 * Florian La Roche, <flla@stud.uni-sb.de> 14 * Charles Hedrick, <hedrick@klinzhai.rutgers.edu> 15 * Linus Torvalds, <torvalds@cs.helsinki.fi> 16 * Alan Cox, <gw4pts@gw4pts.ampr.org> 17 * Matthew Dillon, <dillon@apollo.west.oic.com> 18 * Arnt Gulbrandsen, <agulbra@nvg.unit.no> 19 * Jorge Cwik, <jorge@laser.satlink.net> 20 * 21 * Fixes: 22 * Alan Cox : Numerous verify_area() calls 23 * Alan Cox : Set the ACK bit on a reset 24 * Alan Cox : Stopped it crashing if it closed while 25 * sk->inuse=1 and was trying to connect 26 * (tcp_err()). 27 * Alan Cox : All icmp error handling was broken 28 * pointers passed where wrong and the 29 * socket was looked up backwards. Nobody 30 * tested any icmp error code obviously. 31 * Alan Cox : tcp_err() now handled properly. It 32 * wakes people on errors. poll 33 * behaves and the icmp error race 34 * has gone by moving it into sock.c 35 * Alan Cox : tcp_send_reset() fixed to work for 36 * everything not just packets for 37 * unknown sockets. 38 * Alan Cox : tcp option processing. 39 * Alan Cox : Reset tweaked (still not 100%) [Had 40 * syn rule wrong] 41 * Herp Rosmanith : More reset fixes 42 * Alan Cox : No longer acks invalid rst frames. 43 * Acking any kind of RST is right out. 44 * Alan Cox : Sets an ignore me flag on an rst 45 * receive otherwise odd bits of prattle 46 * escape still 47 * Alan Cox : Fixed another acking RST frame bug. 48 * Should stop LAN workplace lockups. 49 * Alan Cox : Some tidyups using the new skb list 50 * facilities 51 * Alan Cox : sk->keepopen now seems to work 52 * Alan Cox : Pulls options out correctly on accepts 53 * Alan Cox : Fixed assorted sk->rqueue->next errors 54 * Alan Cox : PSH doesn't end a TCP read. Switched a 55 * bit to skb ops. 56 * Alan Cox : Tidied tcp_data to avoid a potential 57 * nasty. 58 * Alan Cox : Added some better commenting, as the 59 * tcp is hard to follow 60 * Alan Cox : Removed incorrect check for 20 * psh 61 * Michael O'Reilly : ack < copied bug fix. 62 * Johannes Stille : Misc tcp fixes (not all in yet). 63 * Alan Cox : FIN with no memory -> CRASH 64 * Alan Cox : Added socket option proto entries. 65 * Also added awareness of them to accept. 66 * Alan Cox : Added TCP options (SOL_TCP) 67 * Alan Cox : Switched wakeup calls to callbacks, 68 * so the kernel can layer network 69 * sockets. 70 * Alan Cox : Use ip_tos/ip_ttl settings. 71 * Alan Cox : Handle FIN (more) properly (we hope). 72 * Alan Cox : RST frames sent on unsynchronised 73 * state ack error. 74 * Alan Cox : Put in missing check for SYN bit. 75 * Alan Cox : Added tcp_select_window() aka NET2E 76 * window non shrink trick. 77 * Alan Cox : Added a couple of small NET2E timer 78 * fixes 79 * Charles Hedrick : TCP fixes 80 * Toomas Tamm : TCP window fixes 81 * Alan Cox : Small URG fix to rlogin ^C ack fight 82 * Charles Hedrick : Rewrote most of it to actually work 83 * Linus : Rewrote tcp_read() and URG handling 84 * completely 85 * Gerhard Koerting: Fixed some missing timer handling 86 * Matthew Dillon : Reworked TCP machine states as per RFC 87 * Gerhard Koerting: PC/TCP workarounds 88 * Adam Caldwell : Assorted timer/timing errors 89 * Matthew Dillon : Fixed another RST bug 90 * Alan Cox : Move to kernel side addressing changes. 91 * Alan Cox : Beginning work on TCP fastpathing 92 * (not yet usable) 93 * Arnt Gulbrandsen: Turbocharged tcp_check() routine. 94 * Alan Cox : TCP fast path debugging 95 * Alan Cox : Window clamping 96 * Michael Riepe : Bug in tcp_check() 97 * Matt Dillon : More TCP improvements and RST bug fixes 98 * Matt Dillon : Yet more small nasties remove from the 99 * TCP code (Be very nice to this man if 100 * tcp finally works 100%) 8) 101 * Alan Cox : BSD accept semantics. 102 * Alan Cox : Reset on closedown bug. 103 * Peter De Schrijver : ENOTCONN check missing in tcp_sendto(). 104 * Michael Pall : Handle poll() after URG properly in 105 * all cases. 106 * Michael Pall : Undo the last fix in tcp_read_urg() 107 * (multi URG PUSH broke rlogin). 108 * Michael Pall : Fix the multi URG PUSH problem in 109 * tcp_readable(), poll() after URG 110 * works now. 111 * Michael Pall : recv(...,MSG_OOB) never blocks in the 112 * BSD api. 113 * Alan Cox : Changed the semantics of sk->socket to 114 * fix a race and a signal problem with 115 * accept() and async I/O. 116 * Alan Cox : Relaxed the rules on tcp_sendto(). 117 * Yury Shevchuk : Really fixed accept() blocking problem. 118 * Craig I. Hagan : Allow for BSD compatible TIME_WAIT for 119 * clients/servers which listen in on 120 * fixed ports. 121 * Alan Cox : Cleaned the above up and shrank it to 122 * a sensible code size. 123 * Alan Cox : Self connect lockup fix. 124 * Alan Cox : No connect to multicast. 125 * Ross Biro : Close unaccepted children on master 126 * socket close. 127 * Alan Cox : Reset tracing code. 128 * Alan Cox : Spurious resets on shutdown. 129 * Alan Cox : Giant 15 minute/60 second timer error 130 * Alan Cox : Small whoops in polling before an 131 * accept. 132 * Alan Cox : Kept the state trace facility since 133 * it's handy for debugging. 134 * Alan Cox : More reset handler fixes. 135 * Alan Cox : Started rewriting the code based on 136 * the RFC's for other useful protocol 137 * references see: Comer, KA9Q NOS, and 138 * for a reference on the difference 139 * between specifications and how BSD 140 * works see the 4.4lite source. 141 * A.N.Kuznetsov : Don't time wait on completion of tidy 142 * close. 143 * Linus Torvalds : Fin/Shutdown & copied_seq changes. 144 * Linus Torvalds : Fixed BSD port reuse to work first syn 145 * Alan Cox : Reimplemented timers as per the RFC 146 * and using multiple timers for sanity. 147 * Alan Cox : Small bug fixes, and a lot of new 148 * comments. 149 * Alan Cox : Fixed dual reader crash by locking 150 * the buffers (much like datagram.c) 151 * Alan Cox : Fixed stuck sockets in probe. A probe 152 * now gets fed up of retrying without 153 * (even a no space) answer. 154 * Alan Cox : Extracted closing code better 155 * Alan Cox : Fixed the closing state machine to 156 * resemble the RFC. 157 * Alan Cox : More 'per spec' fixes. 158 * Jorge Cwik : Even faster checksumming. 159 * Alan Cox : tcp_data() doesn't ack illegal PSH 160 * only frames. At least one pc tcp stack 161 * generates them. 162 * Alan Cox : Cache last socket. 163 * Alan Cox : Per route irtt. 164 * Matt Day : poll()->select() match BSD precisely on error 165 * Alan Cox : New buffers 166 * Marc Tamsky : Various sk->prot->retransmits and 167 * sk->retransmits misupdating fixed. 168 * Fixed tcp_write_timeout: stuck close, 169 * and TCP syn retries gets used now. 170 * Mark Yarvis : In tcp_read_wakeup(), don't send an 171 * ack if state is TCP_CLOSED. 172 * Alan Cox : Look up device on a retransmit - routes may 173 * change. Doesn't yet cope with MSS shrink right 174 * but it's a start! 175 * Marc Tamsky : Closing in closing fixes. 176 * Mike Shaver : RFC1122 verifications. 177 * Alan Cox : rcv_saddr errors. 178 * Alan Cox : Block double connect(). 179 * Alan Cox : Small hooks for enSKIP. 180 * Alexey Kuznetsov: Path MTU discovery. 181 * Alan Cox : Support soft errors. 182 * Alan Cox : Fix MTU discovery pathological case 183 * when the remote claims no mtu! 184 * Marc Tamsky : TCP_CLOSE fix. 185 * Colin (G3TNE) : Send a reset on syn ack replies in 186 * window but wrong (fixes NT lpd problems) 187 * Pedro Roque : Better TCP window handling, delayed ack. 188 * Joerg Reuter : No modification of locked buffers in 189 * tcp_do_retransmit() 190 * Eric Schenk : Changed receiver side silly window 191 * avoidance algorithm to BSD style 192 * algorithm. This doubles throughput 193 * against machines running Solaris, 194 * and seems to result in general 195 * improvement. 196 * Stefan Magdalinski : adjusted tcp_readable() to fix FIONREAD 197 * Willy Konynenberg : Transparent proxying support. 198 * Mike McLagan : Routing by source 199 * Keith Owens : Do proper merging with partial SKB's in 200 * tcp_do_sendmsg to avoid burstiness. 201 * Eric Schenk : Fix fast close down bug with 202 * shutdown() followed by close(). 203 * Andi Kleen : Make poll agree with SIGIO 204 * Salvatore Sanfilippo : Support SO_LINGER with linger == 1 and 205 * lingertime == 0 (RFC 793 ABORT Call) 206 * Hirokazu Takahashi : Use copy_from_user() instead of 207 * csum_and_copy_from_user() if possible. 208 * 209 * Description of States: 210 * 211 * TCP_SYN_SENT sent a connection request, waiting for ack 212 * 213 * TCP_SYN_RECV received a connection request, sent ack, 214 * waiting for final ack in three-way handshake. 215 * 216 * TCP_ESTABLISHED connection established 217 * 218 * TCP_FIN_WAIT1 our side has shutdown, waiting to complete 219 * transmission of remaining buffered data 220 * 221 * TCP_FIN_WAIT2 all buffered data sent, waiting for remote 222 * to shutdown 223 * 224 * TCP_CLOSING both sides have shutdown but we still have 225 * data we have to finish sending 226 * 227 * TCP_TIME_WAIT timeout to catch resent junk before entering 228 * closed, can only be entered from FIN_WAIT2 229 * or CLOSING. Required because the other end 230 * may not have gotten our last ACK causing it 231 * to retransmit the data packet (which we ignore) 232 * 233 * TCP_CLOSE_WAIT remote side has shutdown and is waiting for 234 * us to finish writing our data and to shutdown 235 * (we have to close() to move on to LAST_ACK) 236 * 237 * TCP_LAST_ACK out side has shutdown after remote has 238 * shutdown. There may still be data in our 239 * buffer that we have to finish sending 240 * 241 * TCP_CLOSE socket is finished 242 */ 243 244 #define pr_fmt(fmt) "TCP: " fmt 245 246 #include <crypto/hash.h> 247 #include <linux/kernel.h> 248 #include <linux/module.h> 249 #include <linux/types.h> 250 #include <linux/fcntl.h> 251 #include <linux/poll.h> 252 #include <linux/inet_diag.h> 253 #include <linux/init.h> 254 #include <linux/fs.h> 255 #include <linux/skbuff.h> 256 #include <linux/scatterlist.h> 257 #include <linux/splice.h> 258 #include <linux/net.h> 259 #include <linux/socket.h> 260 #include <linux/random.h> 261 #include <linux/memblock.h> 262 #include <linux/highmem.h> 263 #include <linux/swap.h> 264 #include <linux/cache.h> 265 #include <linux/err.h> 266 #include <linux/time.h> 267 #include <linux/slab.h> 268 #include <linux/errqueue.h> 269 #include <linux/static_key.h> 270 #include <linux/btf.h> 271 272 #include <net/icmp.h> 273 #include <net/inet_common.h> 274 #include <net/tcp.h> 275 #include <net/mptcp.h> 276 #include <net/xfrm.h> 277 #include <net/ip.h> 278 #include <net/sock.h> 279 280 #include <linux/uaccess.h> 281 #include <asm/ioctls.h> 282 #include <net/busy_poll.h> 283 284 /* Track pending CMSGs. */ 285 enum { 286 TCP_CMSG_INQ = 1, 287 TCP_CMSG_TS = 2 288 }; 289 290 struct percpu_counter tcp_orphan_count; 291 EXPORT_SYMBOL_GPL(tcp_orphan_count); 292 293 long sysctl_tcp_mem[3] __read_mostly; 294 EXPORT_SYMBOL(sysctl_tcp_mem); 295 296 atomic_long_t tcp_memory_allocated; /* Current allocated memory. */ 297 EXPORT_SYMBOL(tcp_memory_allocated); 298 299 #if IS_ENABLED(CONFIG_SMC) 300 DEFINE_STATIC_KEY_FALSE(tcp_have_smc); 301 EXPORT_SYMBOL(tcp_have_smc); 302 #endif 303 304 /* 305 * Current number of TCP sockets. 306 */ 307 struct percpu_counter tcp_sockets_allocated; 308 EXPORT_SYMBOL(tcp_sockets_allocated); 309 310 /* 311 * TCP splice context 312 */ 313 struct tcp_splice_state { 314 struct pipe_inode_info *pipe; 315 size_t len; 316 unsigned int flags; 317 }; 318 319 /* 320 * Pressure flag: try to collapse. 321 * Technical note: it is used by multiple contexts non atomically. 322 * All the __sk_mem_schedule() is of this nature: accounting 323 * is strict, actions are advisory and have some latency. 324 */ 325 unsigned long tcp_memory_pressure __read_mostly; 326 EXPORT_SYMBOL_GPL(tcp_memory_pressure); 327 328 DEFINE_STATIC_KEY_FALSE(tcp_rx_skb_cache_key); 329 EXPORT_SYMBOL(tcp_rx_skb_cache_key); 330 331 DEFINE_STATIC_KEY_FALSE(tcp_tx_skb_cache_key); 332 333 void tcp_enter_memory_pressure(struct sock *sk) 334 { 335 unsigned long val; 336 337 if (READ_ONCE(tcp_memory_pressure)) 338 return; 339 val = jiffies; 340 341 if (!val) 342 val--; 343 if (!cmpxchg(&tcp_memory_pressure, 0, val)) 344 NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPMEMORYPRESSURES); 345 } 346 EXPORT_SYMBOL_GPL(tcp_enter_memory_pressure); 347 348 void tcp_leave_memory_pressure(struct sock *sk) 349 { 350 unsigned long val; 351 352 if (!READ_ONCE(tcp_memory_pressure)) 353 return; 354 val = xchg(&tcp_memory_pressure, 0); 355 if (val) 356 NET_ADD_STATS(sock_net(sk), LINUX_MIB_TCPMEMORYPRESSURESCHRONO, 357 jiffies_to_msecs(jiffies - val)); 358 } 359 EXPORT_SYMBOL_GPL(tcp_leave_memory_pressure); 360 361 /* Convert seconds to retransmits based on initial and max timeout */ 362 static u8 secs_to_retrans(int seconds, int timeout, int rto_max) 363 { 364 u8 res = 0; 365 366 if (seconds > 0) { 367 int period = timeout; 368 369 res = 1; 370 while (seconds > period && res < 255) { 371 res++; 372 timeout <<= 1; 373 if (timeout > rto_max) 374 timeout = rto_max; 375 period += timeout; 376 } 377 } 378 return res; 379 } 380 381 /* Convert retransmits to seconds based on initial and max timeout */ 382 static int retrans_to_secs(u8 retrans, int timeout, int rto_max) 383 { 384 int period = 0; 385 386 if (retrans > 0) { 387 period = timeout; 388 while (--retrans) { 389 timeout <<= 1; 390 if (timeout > rto_max) 391 timeout = rto_max; 392 period += timeout; 393 } 394 } 395 return period; 396 } 397 398 static u64 tcp_compute_delivery_rate(const struct tcp_sock *tp) 399 { 400 u32 rate = READ_ONCE(tp->rate_delivered); 401 u32 intv = READ_ONCE(tp->rate_interval_us); 402 u64 rate64 = 0; 403 404 if (rate && intv) { 405 rate64 = (u64)rate * tp->mss_cache * USEC_PER_SEC; 406 do_div(rate64, intv); 407 } 408 return rate64; 409 } 410 411 /* Address-family independent initialization for a tcp_sock. 412 * 413 * NOTE: A lot of things set to zero explicitly by call to 414 * sk_alloc() so need not be done here. 415 */ 416 void tcp_init_sock(struct sock *sk) 417 { 418 struct inet_connection_sock *icsk = inet_csk(sk); 419 struct tcp_sock *tp = tcp_sk(sk); 420 421 tp->out_of_order_queue = RB_ROOT; 422 sk->tcp_rtx_queue = RB_ROOT; 423 tcp_init_xmit_timers(sk); 424 INIT_LIST_HEAD(&tp->tsq_node); 425 INIT_LIST_HEAD(&tp->tsorted_sent_queue); 426 427 icsk->icsk_rto = TCP_TIMEOUT_INIT; 428 icsk->icsk_rto_min = TCP_RTO_MIN; 429 icsk->icsk_delack_max = TCP_DELACK_MAX; 430 tp->mdev_us = jiffies_to_usecs(TCP_TIMEOUT_INIT); 431 minmax_reset(&tp->rtt_min, tcp_jiffies32, ~0U); 432 433 /* So many TCP implementations out there (incorrectly) count the 434 * initial SYN frame in their delayed-ACK and congestion control 435 * algorithms that we must have the following bandaid to talk 436 * efficiently to them. -DaveM 437 */ 438 tp->snd_cwnd = TCP_INIT_CWND; 439 440 /* There's a bubble in the pipe until at least the first ACK. */ 441 tp->app_limited = ~0U; 442 443 /* See draft-stevens-tcpca-spec-01 for discussion of the 444 * initialization of these values. 445 */ 446 tp->snd_ssthresh = TCP_INFINITE_SSTHRESH; 447 tp->snd_cwnd_clamp = ~0; 448 tp->mss_cache = TCP_MSS_DEFAULT; 449 450 tp->reordering = sock_net(sk)->ipv4.sysctl_tcp_reordering; 451 tcp_assign_congestion_control(sk); 452 453 tp->tsoffset = 0; 454 tp->rack.reo_wnd_steps = 1; 455 456 sk->sk_write_space = sk_stream_write_space; 457 sock_set_flag(sk, SOCK_USE_WRITE_QUEUE); 458 459 icsk->icsk_sync_mss = tcp_sync_mss; 460 461 WRITE_ONCE(sk->sk_sndbuf, sock_net(sk)->ipv4.sysctl_tcp_wmem[1]); 462 WRITE_ONCE(sk->sk_rcvbuf, sock_net(sk)->ipv4.sysctl_tcp_rmem[1]); 463 464 sk_sockets_allocated_inc(sk); 465 sk->sk_route_forced_caps = NETIF_F_GSO; 466 } 467 EXPORT_SYMBOL(tcp_init_sock); 468 469 static void tcp_tx_timestamp(struct sock *sk, u16 tsflags) 470 { 471 struct sk_buff *skb = tcp_write_queue_tail(sk); 472 473 if (tsflags && skb) { 474 struct skb_shared_info *shinfo = skb_shinfo(skb); 475 struct tcp_skb_cb *tcb = TCP_SKB_CB(skb); 476 477 sock_tx_timestamp(sk, tsflags, &shinfo->tx_flags); 478 if (tsflags & SOF_TIMESTAMPING_TX_ACK) 479 tcb->txstamp_ack = 1; 480 if (tsflags & SOF_TIMESTAMPING_TX_RECORD_MASK) 481 shinfo->tskey = TCP_SKB_CB(skb)->seq + skb->len - 1; 482 } 483 } 484 485 static bool tcp_stream_is_readable(struct sock *sk, int target) 486 { 487 if (tcp_epollin_ready(sk, target)) 488 return true; 489 490 if (sk->sk_prot->stream_memory_read) 491 return sk->sk_prot->stream_memory_read(sk); 492 return false; 493 } 494 495 /* 496 * Wait for a TCP event. 497 * 498 * Note that we don't need to lock the socket, as the upper poll layers 499 * take care of normal races (between the test and the event) and we don't 500 * go look at any of the socket buffers directly. 501 */ 502 __poll_t tcp_poll(struct file *file, struct socket *sock, poll_table *wait) 503 { 504 __poll_t mask; 505 struct sock *sk = sock->sk; 506 const struct tcp_sock *tp = tcp_sk(sk); 507 int state; 508 509 sock_poll_wait(file, sock, wait); 510 511 state = inet_sk_state_load(sk); 512 if (state == TCP_LISTEN) 513 return inet_csk_listen_poll(sk); 514 515 /* Socket is not locked. We are protected from async events 516 * by poll logic and correct handling of state changes 517 * made by other threads is impossible in any case. 518 */ 519 520 mask = 0; 521 522 /* 523 * EPOLLHUP is certainly not done right. But poll() doesn't 524 * have a notion of HUP in just one direction, and for a 525 * socket the read side is more interesting. 526 * 527 * Some poll() documentation says that EPOLLHUP is incompatible 528 * with the EPOLLOUT/POLLWR flags, so somebody should check this 529 * all. But careful, it tends to be safer to return too many 530 * bits than too few, and you can easily break real applications 531 * if you don't tell them that something has hung up! 532 * 533 * Check-me. 534 * 535 * Check number 1. EPOLLHUP is _UNMASKABLE_ event (see UNIX98 and 536 * our fs/select.c). It means that after we received EOF, 537 * poll always returns immediately, making impossible poll() on write() 538 * in state CLOSE_WAIT. One solution is evident --- to set EPOLLHUP 539 * if and only if shutdown has been made in both directions. 540 * Actually, it is interesting to look how Solaris and DUX 541 * solve this dilemma. I would prefer, if EPOLLHUP were maskable, 542 * then we could set it on SND_SHUTDOWN. BTW examples given 543 * in Stevens' books assume exactly this behaviour, it explains 544 * why EPOLLHUP is incompatible with EPOLLOUT. --ANK 545 * 546 * NOTE. Check for TCP_CLOSE is added. The goal is to prevent 547 * blocking on fresh not-connected or disconnected socket. --ANK 548 */ 549 if (sk->sk_shutdown == SHUTDOWN_MASK || state == TCP_CLOSE) 550 mask |= EPOLLHUP; 551 if (sk->sk_shutdown & RCV_SHUTDOWN) 552 mask |= EPOLLIN | EPOLLRDNORM | EPOLLRDHUP; 553 554 /* Connected or passive Fast Open socket? */ 555 if (state != TCP_SYN_SENT && 556 (state != TCP_SYN_RECV || rcu_access_pointer(tp->fastopen_rsk))) { 557 int target = sock_rcvlowat(sk, 0, INT_MAX); 558 559 if (READ_ONCE(tp->urg_seq) == READ_ONCE(tp->copied_seq) && 560 !sock_flag(sk, SOCK_URGINLINE) && 561 tp->urg_data) 562 target++; 563 564 if (tcp_stream_is_readable(sk, target)) 565 mask |= EPOLLIN | EPOLLRDNORM; 566 567 if (!(sk->sk_shutdown & SEND_SHUTDOWN)) { 568 if (__sk_stream_is_writeable(sk, 1)) { 569 mask |= EPOLLOUT | EPOLLWRNORM; 570 } else { /* send SIGIO later */ 571 sk_set_bit(SOCKWQ_ASYNC_NOSPACE, sk); 572 set_bit(SOCK_NOSPACE, &sk->sk_socket->flags); 573 574 /* Race breaker. If space is freed after 575 * wspace test but before the flags are set, 576 * IO signal will be lost. Memory barrier 577 * pairs with the input side. 578 */ 579 smp_mb__after_atomic(); 580 if (__sk_stream_is_writeable(sk, 1)) 581 mask |= EPOLLOUT | EPOLLWRNORM; 582 } 583 } else 584 mask |= EPOLLOUT | EPOLLWRNORM; 585 586 if (tp->urg_data & TCP_URG_VALID) 587 mask |= EPOLLPRI; 588 } else if (state == TCP_SYN_SENT && inet_sk(sk)->defer_connect) { 589 /* Active TCP fastopen socket with defer_connect 590 * Return EPOLLOUT so application can call write() 591 * in order for kernel to generate SYN+data 592 */ 593 mask |= EPOLLOUT | EPOLLWRNORM; 594 } 595 /* This barrier is coupled with smp_wmb() in tcp_reset() */ 596 smp_rmb(); 597 if (sk->sk_err || !skb_queue_empty_lockless(&sk->sk_error_queue)) 598 mask |= EPOLLERR; 599 600 return mask; 601 } 602 EXPORT_SYMBOL(tcp_poll); 603 604 int tcp_ioctl(struct sock *sk, int cmd, unsigned long arg) 605 { 606 struct tcp_sock *tp = tcp_sk(sk); 607 int answ; 608 bool slow; 609 610 switch (cmd) { 611 case SIOCINQ: 612 if (sk->sk_state == TCP_LISTEN) 613 return -EINVAL; 614 615 slow = lock_sock_fast(sk); 616 answ = tcp_inq(sk); 617 unlock_sock_fast(sk, slow); 618 break; 619 case SIOCATMARK: 620 answ = tp->urg_data && 621 READ_ONCE(tp->urg_seq) == READ_ONCE(tp->copied_seq); 622 break; 623 case SIOCOUTQ: 624 if (sk->sk_state == TCP_LISTEN) 625 return -EINVAL; 626 627 if ((1 << sk->sk_state) & (TCPF_SYN_SENT | TCPF_SYN_RECV)) 628 answ = 0; 629 else 630 answ = READ_ONCE(tp->write_seq) - tp->snd_una; 631 break; 632 case SIOCOUTQNSD: 633 if (sk->sk_state == TCP_LISTEN) 634 return -EINVAL; 635 636 if ((1 << sk->sk_state) & (TCPF_SYN_SENT | TCPF_SYN_RECV)) 637 answ = 0; 638 else 639 answ = READ_ONCE(tp->write_seq) - 640 READ_ONCE(tp->snd_nxt); 641 break; 642 default: 643 return -ENOIOCTLCMD; 644 } 645 646 return put_user(answ, (int __user *)arg); 647 } 648 EXPORT_SYMBOL(tcp_ioctl); 649 650 static inline void tcp_mark_push(struct tcp_sock *tp, struct sk_buff *skb) 651 { 652 TCP_SKB_CB(skb)->tcp_flags |= TCPHDR_PSH; 653 tp->pushed_seq = tp->write_seq; 654 } 655 656 static inline bool forced_push(const struct tcp_sock *tp) 657 { 658 return after(tp->write_seq, tp->pushed_seq + (tp->max_window >> 1)); 659 } 660 661 static void skb_entail(struct sock *sk, struct sk_buff *skb) 662 { 663 struct tcp_sock *tp = tcp_sk(sk); 664 struct tcp_skb_cb *tcb = TCP_SKB_CB(skb); 665 666 skb->csum = 0; 667 tcb->seq = tcb->end_seq = tp->write_seq; 668 tcb->tcp_flags = TCPHDR_ACK; 669 tcb->sacked = 0; 670 __skb_header_release(skb); 671 tcp_add_write_queue_tail(sk, skb); 672 sk_wmem_queued_add(sk, skb->truesize); 673 sk_mem_charge(sk, skb->truesize); 674 if (tp->nonagle & TCP_NAGLE_PUSH) 675 tp->nonagle &= ~TCP_NAGLE_PUSH; 676 677 tcp_slow_start_after_idle_check(sk); 678 } 679 680 static inline void tcp_mark_urg(struct tcp_sock *tp, int flags) 681 { 682 if (flags & MSG_OOB) 683 tp->snd_up = tp->write_seq; 684 } 685 686 /* If a not yet filled skb is pushed, do not send it if 687 * we have data packets in Qdisc or NIC queues : 688 * Because TX completion will happen shortly, it gives a chance 689 * to coalesce future sendmsg() payload into this skb, without 690 * need for a timer, and with no latency trade off. 691 * As packets containing data payload have a bigger truesize 692 * than pure acks (dataless) packets, the last checks prevent 693 * autocorking if we only have an ACK in Qdisc/NIC queues, 694 * or if TX completion was delayed after we processed ACK packet. 695 */ 696 static bool tcp_should_autocork(struct sock *sk, struct sk_buff *skb, 697 int size_goal) 698 { 699 return skb->len < size_goal && 700 sock_net(sk)->ipv4.sysctl_tcp_autocorking && 701 !tcp_rtx_queue_empty(sk) && 702 refcount_read(&sk->sk_wmem_alloc) > skb->truesize; 703 } 704 705 void tcp_push(struct sock *sk, int flags, int mss_now, 706 int nonagle, int size_goal) 707 { 708 struct tcp_sock *tp = tcp_sk(sk); 709 struct sk_buff *skb; 710 711 skb = tcp_write_queue_tail(sk); 712 if (!skb) 713 return; 714 if (!(flags & MSG_MORE) || forced_push(tp)) 715 tcp_mark_push(tp, skb); 716 717 tcp_mark_urg(tp, flags); 718 719 if (tcp_should_autocork(sk, skb, size_goal)) { 720 721 /* avoid atomic op if TSQ_THROTTLED bit is already set */ 722 if (!test_bit(TSQ_THROTTLED, &sk->sk_tsq_flags)) { 723 NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPAUTOCORKING); 724 set_bit(TSQ_THROTTLED, &sk->sk_tsq_flags); 725 } 726 /* It is possible TX completion already happened 727 * before we set TSQ_THROTTLED. 728 */ 729 if (refcount_read(&sk->sk_wmem_alloc) > skb->truesize) 730 return; 731 } 732 733 if (flags & MSG_MORE) 734 nonagle = TCP_NAGLE_CORK; 735 736 __tcp_push_pending_frames(sk, mss_now, nonagle); 737 } 738 739 static int tcp_splice_data_recv(read_descriptor_t *rd_desc, struct sk_buff *skb, 740 unsigned int offset, size_t len) 741 { 742 struct tcp_splice_state *tss = rd_desc->arg.data; 743 int ret; 744 745 ret = skb_splice_bits(skb, skb->sk, offset, tss->pipe, 746 min(rd_desc->count, len), tss->flags); 747 if (ret > 0) 748 rd_desc->count -= ret; 749 return ret; 750 } 751 752 static int __tcp_splice_read(struct sock *sk, struct tcp_splice_state *tss) 753 { 754 /* Store TCP splice context information in read_descriptor_t. */ 755 read_descriptor_t rd_desc = { 756 .arg.data = tss, 757 .count = tss->len, 758 }; 759 760 return tcp_read_sock(sk, &rd_desc, tcp_splice_data_recv); 761 } 762 763 /** 764 * tcp_splice_read - splice data from TCP socket to a pipe 765 * @sock: socket to splice from 766 * @ppos: position (not valid) 767 * @pipe: pipe to splice to 768 * @len: number of bytes to splice 769 * @flags: splice modifier flags 770 * 771 * Description: 772 * Will read pages from given socket and fill them into a pipe. 773 * 774 **/ 775 ssize_t tcp_splice_read(struct socket *sock, loff_t *ppos, 776 struct pipe_inode_info *pipe, size_t len, 777 unsigned int flags) 778 { 779 struct sock *sk = sock->sk; 780 struct tcp_splice_state tss = { 781 .pipe = pipe, 782 .len = len, 783 .flags = flags, 784 }; 785 long timeo; 786 ssize_t spliced; 787 int ret; 788 789 sock_rps_record_flow(sk); 790 /* 791 * We can't seek on a socket input 792 */ 793 if (unlikely(*ppos)) 794 return -ESPIPE; 795 796 ret = spliced = 0; 797 798 lock_sock(sk); 799 800 timeo = sock_rcvtimeo(sk, sock->file->f_flags & O_NONBLOCK); 801 while (tss.len) { 802 ret = __tcp_splice_read(sk, &tss); 803 if (ret < 0) 804 break; 805 else if (!ret) { 806 if (spliced) 807 break; 808 if (sock_flag(sk, SOCK_DONE)) 809 break; 810 if (sk->sk_err) { 811 ret = sock_error(sk); 812 break; 813 } 814 if (sk->sk_shutdown & RCV_SHUTDOWN) 815 break; 816 if (sk->sk_state == TCP_CLOSE) { 817 /* 818 * This occurs when user tries to read 819 * from never connected socket. 820 */ 821 ret = -ENOTCONN; 822 break; 823 } 824 if (!timeo) { 825 ret = -EAGAIN; 826 break; 827 } 828 /* if __tcp_splice_read() got nothing while we have 829 * an skb in receive queue, we do not want to loop. 830 * This might happen with URG data. 831 */ 832 if (!skb_queue_empty(&sk->sk_receive_queue)) 833 break; 834 sk_wait_data(sk, &timeo, NULL); 835 if (signal_pending(current)) { 836 ret = sock_intr_errno(timeo); 837 break; 838 } 839 continue; 840 } 841 tss.len -= ret; 842 spliced += ret; 843 844 if (!timeo) 845 break; 846 release_sock(sk); 847 lock_sock(sk); 848 849 if (sk->sk_err || sk->sk_state == TCP_CLOSE || 850 (sk->sk_shutdown & RCV_SHUTDOWN) || 851 signal_pending(current)) 852 break; 853 } 854 855 release_sock(sk); 856 857 if (spliced) 858 return spliced; 859 860 return ret; 861 } 862 EXPORT_SYMBOL(tcp_splice_read); 863 864 struct sk_buff *sk_stream_alloc_skb(struct sock *sk, int size, gfp_t gfp, 865 bool force_schedule) 866 { 867 struct sk_buff *skb; 868 869 if (likely(!size)) { 870 skb = sk->sk_tx_skb_cache; 871 if (skb) { 872 skb->truesize = SKB_TRUESIZE(skb_end_offset(skb)); 873 sk->sk_tx_skb_cache = NULL; 874 pskb_trim(skb, 0); 875 INIT_LIST_HEAD(&skb->tcp_tsorted_anchor); 876 skb_shinfo(skb)->tx_flags = 0; 877 memset(TCP_SKB_CB(skb), 0, sizeof(struct tcp_skb_cb)); 878 return skb; 879 } 880 } 881 /* The TCP header must be at least 32-bit aligned. */ 882 size = ALIGN(size, 4); 883 884 if (unlikely(tcp_under_memory_pressure(sk))) 885 sk_mem_reclaim_partial(sk); 886 887 skb = alloc_skb_fclone(size + sk->sk_prot->max_header, gfp); 888 if (likely(skb)) { 889 bool mem_scheduled; 890 891 if (force_schedule) { 892 mem_scheduled = true; 893 sk_forced_mem_schedule(sk, skb->truesize); 894 } else { 895 mem_scheduled = sk_wmem_schedule(sk, skb->truesize); 896 } 897 if (likely(mem_scheduled)) { 898 skb_reserve(skb, sk->sk_prot->max_header); 899 /* 900 * Make sure that we have exactly size bytes 901 * available to the caller, no more, no less. 902 */ 903 skb->reserved_tailroom = skb->end - skb->tail - size; 904 INIT_LIST_HEAD(&skb->tcp_tsorted_anchor); 905 return skb; 906 } 907 __kfree_skb(skb); 908 } else { 909 sk->sk_prot->enter_memory_pressure(sk); 910 sk_stream_moderate_sndbuf(sk); 911 } 912 return NULL; 913 } 914 915 static unsigned int tcp_xmit_size_goal(struct sock *sk, u32 mss_now, 916 int large_allowed) 917 { 918 struct tcp_sock *tp = tcp_sk(sk); 919 u32 new_size_goal, size_goal; 920 921 if (!large_allowed) 922 return mss_now; 923 924 /* Note : tcp_tso_autosize() will eventually split this later */ 925 new_size_goal = sk->sk_gso_max_size - 1 - MAX_TCP_HEADER; 926 new_size_goal = tcp_bound_to_half_wnd(tp, new_size_goal); 927 928 /* We try hard to avoid divides here */ 929 size_goal = tp->gso_segs * mss_now; 930 if (unlikely(new_size_goal < size_goal || 931 new_size_goal >= size_goal + mss_now)) { 932 tp->gso_segs = min_t(u16, new_size_goal / mss_now, 933 sk->sk_gso_max_segs); 934 size_goal = tp->gso_segs * mss_now; 935 } 936 937 return max(size_goal, mss_now); 938 } 939 940 int tcp_send_mss(struct sock *sk, int *size_goal, int flags) 941 { 942 int mss_now; 943 944 mss_now = tcp_current_mss(sk); 945 *size_goal = tcp_xmit_size_goal(sk, mss_now, !(flags & MSG_OOB)); 946 947 return mss_now; 948 } 949 950 /* In some cases, both sendpage() and sendmsg() could have added 951 * an skb to the write queue, but failed adding payload on it. 952 * We need to remove it to consume less memory, but more 953 * importantly be able to generate EPOLLOUT for Edge Trigger epoll() 954 * users. 955 */ 956 void tcp_remove_empty_skb(struct sock *sk, struct sk_buff *skb) 957 { 958 if (skb && !skb->len) { 959 tcp_unlink_write_queue(skb, sk); 960 if (tcp_write_queue_empty(sk)) 961 tcp_chrono_stop(sk, TCP_CHRONO_BUSY); 962 sk_wmem_free_skb(sk, skb); 963 } 964 } 965 966 struct sk_buff *tcp_build_frag(struct sock *sk, int size_goal, int flags, 967 struct page *page, int offset, size_t *size) 968 { 969 struct sk_buff *skb = tcp_write_queue_tail(sk); 970 struct tcp_sock *tp = tcp_sk(sk); 971 bool can_coalesce; 972 int copy, i; 973 974 if (!skb || (copy = size_goal - skb->len) <= 0 || 975 !tcp_skb_can_collapse_to(skb)) { 976 new_segment: 977 if (!sk_stream_memory_free(sk)) 978 return NULL; 979 980 skb = sk_stream_alloc_skb(sk, 0, sk->sk_allocation, 981 tcp_rtx_and_write_queues_empty(sk)); 982 if (!skb) 983 return NULL; 984 985 #ifdef CONFIG_TLS_DEVICE 986 skb->decrypted = !!(flags & MSG_SENDPAGE_DECRYPTED); 987 #endif 988 skb_entail(sk, skb); 989 copy = size_goal; 990 } 991 992 if (copy > *size) 993 copy = *size; 994 995 i = skb_shinfo(skb)->nr_frags; 996 can_coalesce = skb_can_coalesce(skb, i, page, offset); 997 if (!can_coalesce && i >= sysctl_max_skb_frags) { 998 tcp_mark_push(tp, skb); 999 goto new_segment; 1000 } 1001 if (!sk_wmem_schedule(sk, copy)) 1002 return NULL; 1003 1004 if (can_coalesce) { 1005 skb_frag_size_add(&skb_shinfo(skb)->frags[i - 1], copy); 1006 } else { 1007 get_page(page); 1008 skb_fill_page_desc(skb, i, page, offset, copy); 1009 } 1010 1011 if (!(flags & MSG_NO_SHARED_FRAGS)) 1012 skb_shinfo(skb)->flags |= SKBFL_SHARED_FRAG; 1013 1014 skb->len += copy; 1015 skb->data_len += copy; 1016 skb->truesize += copy; 1017 sk_wmem_queued_add(sk, copy); 1018 sk_mem_charge(sk, copy); 1019 skb->ip_summed = CHECKSUM_PARTIAL; 1020 WRITE_ONCE(tp->write_seq, tp->write_seq + copy); 1021 TCP_SKB_CB(skb)->end_seq += copy; 1022 tcp_skb_pcount_set(skb, 0); 1023 1024 *size = copy; 1025 return skb; 1026 } 1027 1028 ssize_t do_tcp_sendpages(struct sock *sk, struct page *page, int offset, 1029 size_t size, int flags) 1030 { 1031 struct tcp_sock *tp = tcp_sk(sk); 1032 int mss_now, size_goal; 1033 int err; 1034 ssize_t copied; 1035 long timeo = sock_sndtimeo(sk, flags & MSG_DONTWAIT); 1036 1037 if (IS_ENABLED(CONFIG_DEBUG_VM) && 1038 WARN_ONCE(!sendpage_ok(page), 1039 "page must not be a Slab one and have page_count > 0")) 1040 return -EINVAL; 1041 1042 /* Wait for a connection to finish. One exception is TCP Fast Open 1043 * (passive side) where data is allowed to be sent before a connection 1044 * is fully established. 1045 */ 1046 if (((1 << sk->sk_state) & ~(TCPF_ESTABLISHED | TCPF_CLOSE_WAIT)) && 1047 !tcp_passive_fastopen(sk)) { 1048 err = sk_stream_wait_connect(sk, &timeo); 1049 if (err != 0) 1050 goto out_err; 1051 } 1052 1053 sk_clear_bit(SOCKWQ_ASYNC_NOSPACE, sk); 1054 1055 mss_now = tcp_send_mss(sk, &size_goal, flags); 1056 copied = 0; 1057 1058 err = -EPIPE; 1059 if (sk->sk_err || (sk->sk_shutdown & SEND_SHUTDOWN)) 1060 goto out_err; 1061 1062 while (size > 0) { 1063 struct sk_buff *skb; 1064 size_t copy = size; 1065 1066 skb = tcp_build_frag(sk, size_goal, flags, page, offset, ©); 1067 if (!skb) 1068 goto wait_for_space; 1069 1070 if (!copied) 1071 TCP_SKB_CB(skb)->tcp_flags &= ~TCPHDR_PSH; 1072 1073 copied += copy; 1074 offset += copy; 1075 size -= copy; 1076 if (!size) 1077 goto out; 1078 1079 if (skb->len < size_goal || (flags & MSG_OOB)) 1080 continue; 1081 1082 if (forced_push(tp)) { 1083 tcp_mark_push(tp, skb); 1084 __tcp_push_pending_frames(sk, mss_now, TCP_NAGLE_PUSH); 1085 } else if (skb == tcp_send_head(sk)) 1086 tcp_push_one(sk, mss_now); 1087 continue; 1088 1089 wait_for_space: 1090 set_bit(SOCK_NOSPACE, &sk->sk_socket->flags); 1091 tcp_push(sk, flags & ~MSG_MORE, mss_now, 1092 TCP_NAGLE_PUSH, size_goal); 1093 1094 err = sk_stream_wait_memory(sk, &timeo); 1095 if (err != 0) 1096 goto do_error; 1097 1098 mss_now = tcp_send_mss(sk, &size_goal, flags); 1099 } 1100 1101 out: 1102 if (copied) { 1103 tcp_tx_timestamp(sk, sk->sk_tsflags); 1104 if (!(flags & MSG_SENDPAGE_NOTLAST)) 1105 tcp_push(sk, flags, mss_now, tp->nonagle, size_goal); 1106 } 1107 return copied; 1108 1109 do_error: 1110 tcp_remove_empty_skb(sk, tcp_write_queue_tail(sk)); 1111 if (copied) 1112 goto out; 1113 out_err: 1114 /* make sure we wake any epoll edge trigger waiter */ 1115 if (unlikely(tcp_rtx_and_write_queues_empty(sk) && err == -EAGAIN)) { 1116 sk->sk_write_space(sk); 1117 tcp_chrono_stop(sk, TCP_CHRONO_SNDBUF_LIMITED); 1118 } 1119 return sk_stream_error(sk, flags, err); 1120 } 1121 EXPORT_SYMBOL_GPL(do_tcp_sendpages); 1122 1123 int tcp_sendpage_locked(struct sock *sk, struct page *page, int offset, 1124 size_t size, int flags) 1125 { 1126 if (!(sk->sk_route_caps & NETIF_F_SG)) 1127 return sock_no_sendpage_locked(sk, page, offset, size, flags); 1128 1129 tcp_rate_check_app_limited(sk); /* is sending application-limited? */ 1130 1131 return do_tcp_sendpages(sk, page, offset, size, flags); 1132 } 1133 EXPORT_SYMBOL_GPL(tcp_sendpage_locked); 1134 1135 int tcp_sendpage(struct sock *sk, struct page *page, int offset, 1136 size_t size, int flags) 1137 { 1138 int ret; 1139 1140 lock_sock(sk); 1141 ret = tcp_sendpage_locked(sk, page, offset, size, flags); 1142 release_sock(sk); 1143 1144 return ret; 1145 } 1146 EXPORT_SYMBOL(tcp_sendpage); 1147 1148 void tcp_free_fastopen_req(struct tcp_sock *tp) 1149 { 1150 if (tp->fastopen_req) { 1151 kfree(tp->fastopen_req); 1152 tp->fastopen_req = NULL; 1153 } 1154 } 1155 1156 static int tcp_sendmsg_fastopen(struct sock *sk, struct msghdr *msg, 1157 int *copied, size_t size, 1158 struct ubuf_info *uarg) 1159 { 1160 struct tcp_sock *tp = tcp_sk(sk); 1161 struct inet_sock *inet = inet_sk(sk); 1162 struct sockaddr *uaddr = msg->msg_name; 1163 int err, flags; 1164 1165 if (!(sock_net(sk)->ipv4.sysctl_tcp_fastopen & TFO_CLIENT_ENABLE) || 1166 (uaddr && msg->msg_namelen >= sizeof(uaddr->sa_family) && 1167 uaddr->sa_family == AF_UNSPEC)) 1168 return -EOPNOTSUPP; 1169 if (tp->fastopen_req) 1170 return -EALREADY; /* Another Fast Open is in progress */ 1171 1172 tp->fastopen_req = kzalloc(sizeof(struct tcp_fastopen_request), 1173 sk->sk_allocation); 1174 if (unlikely(!tp->fastopen_req)) 1175 return -ENOBUFS; 1176 tp->fastopen_req->data = msg; 1177 tp->fastopen_req->size = size; 1178 tp->fastopen_req->uarg = uarg; 1179 1180 if (inet->defer_connect) { 1181 err = tcp_connect(sk); 1182 /* Same failure procedure as in tcp_v4/6_connect */ 1183 if (err) { 1184 tcp_set_state(sk, TCP_CLOSE); 1185 inet->inet_dport = 0; 1186 sk->sk_route_caps = 0; 1187 } 1188 } 1189 flags = (msg->msg_flags & MSG_DONTWAIT) ? O_NONBLOCK : 0; 1190 err = __inet_stream_connect(sk->sk_socket, uaddr, 1191 msg->msg_namelen, flags, 1); 1192 /* fastopen_req could already be freed in __inet_stream_connect 1193 * if the connection times out or gets rst 1194 */ 1195 if (tp->fastopen_req) { 1196 *copied = tp->fastopen_req->copied; 1197 tcp_free_fastopen_req(tp); 1198 inet->defer_connect = 0; 1199 } 1200 return err; 1201 } 1202 1203 int tcp_sendmsg_locked(struct sock *sk, struct msghdr *msg, size_t size) 1204 { 1205 struct tcp_sock *tp = tcp_sk(sk); 1206 struct ubuf_info *uarg = NULL; 1207 struct sk_buff *skb; 1208 struct sockcm_cookie sockc; 1209 int flags, err, copied = 0; 1210 int mss_now = 0, size_goal, copied_syn = 0; 1211 int process_backlog = 0; 1212 bool zc = false; 1213 long timeo; 1214 1215 flags = msg->msg_flags; 1216 1217 if (flags & MSG_ZEROCOPY && size && sock_flag(sk, SOCK_ZEROCOPY)) { 1218 skb = tcp_write_queue_tail(sk); 1219 uarg = msg_zerocopy_realloc(sk, size, skb_zcopy(skb)); 1220 if (!uarg) { 1221 err = -ENOBUFS; 1222 goto out_err; 1223 } 1224 1225 zc = sk->sk_route_caps & NETIF_F_SG; 1226 if (!zc) 1227 uarg->zerocopy = 0; 1228 } 1229 1230 if (unlikely(flags & MSG_FASTOPEN || inet_sk(sk)->defer_connect) && 1231 !tp->repair) { 1232 err = tcp_sendmsg_fastopen(sk, msg, &copied_syn, size, uarg); 1233 if (err == -EINPROGRESS && copied_syn > 0) 1234 goto out; 1235 else if (err) 1236 goto out_err; 1237 } 1238 1239 timeo = sock_sndtimeo(sk, flags & MSG_DONTWAIT); 1240 1241 tcp_rate_check_app_limited(sk); /* is sending application-limited? */ 1242 1243 /* Wait for a connection to finish. One exception is TCP Fast Open 1244 * (passive side) where data is allowed to be sent before a connection 1245 * is fully established. 1246 */ 1247 if (((1 << sk->sk_state) & ~(TCPF_ESTABLISHED | TCPF_CLOSE_WAIT)) && 1248 !tcp_passive_fastopen(sk)) { 1249 err = sk_stream_wait_connect(sk, &timeo); 1250 if (err != 0) 1251 goto do_error; 1252 } 1253 1254 if (unlikely(tp->repair)) { 1255 if (tp->repair_queue == TCP_RECV_QUEUE) { 1256 copied = tcp_send_rcvq(sk, msg, size); 1257 goto out_nopush; 1258 } 1259 1260 err = -EINVAL; 1261 if (tp->repair_queue == TCP_NO_QUEUE) 1262 goto out_err; 1263 1264 /* 'common' sending to sendq */ 1265 } 1266 1267 sockcm_init(&sockc, sk); 1268 if (msg->msg_controllen) { 1269 err = sock_cmsg_send(sk, msg, &sockc); 1270 if (unlikely(err)) { 1271 err = -EINVAL; 1272 goto out_err; 1273 } 1274 } 1275 1276 /* This should be in poll */ 1277 sk_clear_bit(SOCKWQ_ASYNC_NOSPACE, sk); 1278 1279 /* Ok commence sending. */ 1280 copied = 0; 1281 1282 restart: 1283 mss_now = tcp_send_mss(sk, &size_goal, flags); 1284 1285 err = -EPIPE; 1286 if (sk->sk_err || (sk->sk_shutdown & SEND_SHUTDOWN)) 1287 goto do_error; 1288 1289 while (msg_data_left(msg)) { 1290 int copy = 0; 1291 1292 skb = tcp_write_queue_tail(sk); 1293 if (skb) 1294 copy = size_goal - skb->len; 1295 1296 if (copy <= 0 || !tcp_skb_can_collapse_to(skb)) { 1297 bool first_skb; 1298 1299 new_segment: 1300 if (!sk_stream_memory_free(sk)) 1301 goto wait_for_space; 1302 1303 if (unlikely(process_backlog >= 16)) { 1304 process_backlog = 0; 1305 if (sk_flush_backlog(sk)) 1306 goto restart; 1307 } 1308 first_skb = tcp_rtx_and_write_queues_empty(sk); 1309 skb = sk_stream_alloc_skb(sk, 0, sk->sk_allocation, 1310 first_skb); 1311 if (!skb) 1312 goto wait_for_space; 1313 1314 process_backlog++; 1315 skb->ip_summed = CHECKSUM_PARTIAL; 1316 1317 skb_entail(sk, skb); 1318 copy = size_goal; 1319 1320 /* All packets are restored as if they have 1321 * already been sent. skb_mstamp_ns isn't set to 1322 * avoid wrong rtt estimation. 1323 */ 1324 if (tp->repair) 1325 TCP_SKB_CB(skb)->sacked |= TCPCB_REPAIRED; 1326 } 1327 1328 /* Try to append data to the end of skb. */ 1329 if (copy > msg_data_left(msg)) 1330 copy = msg_data_left(msg); 1331 1332 /* Where to copy to? */ 1333 if (skb_availroom(skb) > 0 && !zc) { 1334 /* We have some space in skb head. Superb! */ 1335 copy = min_t(int, copy, skb_availroom(skb)); 1336 err = skb_add_data_nocache(sk, skb, &msg->msg_iter, copy); 1337 if (err) 1338 goto do_fault; 1339 } else if (!zc) { 1340 bool merge = true; 1341 int i = skb_shinfo(skb)->nr_frags; 1342 struct page_frag *pfrag = sk_page_frag(sk); 1343 1344 if (!sk_page_frag_refill(sk, pfrag)) 1345 goto wait_for_space; 1346 1347 if (!skb_can_coalesce(skb, i, pfrag->page, 1348 pfrag->offset)) { 1349 if (i >= sysctl_max_skb_frags) { 1350 tcp_mark_push(tp, skb); 1351 goto new_segment; 1352 } 1353 merge = false; 1354 } 1355 1356 copy = min_t(int, copy, pfrag->size - pfrag->offset); 1357 1358 if (!sk_wmem_schedule(sk, copy)) 1359 goto wait_for_space; 1360 1361 err = skb_copy_to_page_nocache(sk, &msg->msg_iter, skb, 1362 pfrag->page, 1363 pfrag->offset, 1364 copy); 1365 if (err) 1366 goto do_error; 1367 1368 /* Update the skb. */ 1369 if (merge) { 1370 skb_frag_size_add(&skb_shinfo(skb)->frags[i - 1], copy); 1371 } else { 1372 skb_fill_page_desc(skb, i, pfrag->page, 1373 pfrag->offset, copy); 1374 page_ref_inc(pfrag->page); 1375 } 1376 pfrag->offset += copy; 1377 } else { 1378 if (!sk_wmem_schedule(sk, copy)) 1379 goto wait_for_space; 1380 1381 err = skb_zerocopy_iter_stream(sk, skb, msg, copy, uarg); 1382 if (err == -EMSGSIZE || err == -EEXIST) { 1383 tcp_mark_push(tp, skb); 1384 goto new_segment; 1385 } 1386 if (err < 0) 1387 goto do_error; 1388 copy = err; 1389 } 1390 1391 if (!copied) 1392 TCP_SKB_CB(skb)->tcp_flags &= ~TCPHDR_PSH; 1393 1394 WRITE_ONCE(tp->write_seq, tp->write_seq + copy); 1395 TCP_SKB_CB(skb)->end_seq += copy; 1396 tcp_skb_pcount_set(skb, 0); 1397 1398 copied += copy; 1399 if (!msg_data_left(msg)) { 1400 if (unlikely(flags & MSG_EOR)) 1401 TCP_SKB_CB(skb)->eor = 1; 1402 goto out; 1403 } 1404 1405 if (skb->len < size_goal || (flags & MSG_OOB) || unlikely(tp->repair)) 1406 continue; 1407 1408 if (forced_push(tp)) { 1409 tcp_mark_push(tp, skb); 1410 __tcp_push_pending_frames(sk, mss_now, TCP_NAGLE_PUSH); 1411 } else if (skb == tcp_send_head(sk)) 1412 tcp_push_one(sk, mss_now); 1413 continue; 1414 1415 wait_for_space: 1416 set_bit(SOCK_NOSPACE, &sk->sk_socket->flags); 1417 if (copied) 1418 tcp_push(sk, flags & ~MSG_MORE, mss_now, 1419 TCP_NAGLE_PUSH, size_goal); 1420 1421 err = sk_stream_wait_memory(sk, &timeo); 1422 if (err != 0) 1423 goto do_error; 1424 1425 mss_now = tcp_send_mss(sk, &size_goal, flags); 1426 } 1427 1428 out: 1429 if (copied) { 1430 tcp_tx_timestamp(sk, sockc.tsflags); 1431 tcp_push(sk, flags, mss_now, tp->nonagle, size_goal); 1432 } 1433 out_nopush: 1434 net_zcopy_put(uarg); 1435 return copied + copied_syn; 1436 1437 do_error: 1438 skb = tcp_write_queue_tail(sk); 1439 do_fault: 1440 tcp_remove_empty_skb(sk, skb); 1441 1442 if (copied + copied_syn) 1443 goto out; 1444 out_err: 1445 net_zcopy_put_abort(uarg, true); 1446 err = sk_stream_error(sk, flags, err); 1447 /* make sure we wake any epoll edge trigger waiter */ 1448 if (unlikely(tcp_rtx_and_write_queues_empty(sk) && err == -EAGAIN)) { 1449 sk->sk_write_space(sk); 1450 tcp_chrono_stop(sk, TCP_CHRONO_SNDBUF_LIMITED); 1451 } 1452 return err; 1453 } 1454 EXPORT_SYMBOL_GPL(tcp_sendmsg_locked); 1455 1456 int tcp_sendmsg(struct sock *sk, struct msghdr *msg, size_t size) 1457 { 1458 int ret; 1459 1460 lock_sock(sk); 1461 ret = tcp_sendmsg_locked(sk, msg, size); 1462 release_sock(sk); 1463 1464 return ret; 1465 } 1466 EXPORT_SYMBOL(tcp_sendmsg); 1467 1468 /* 1469 * Handle reading urgent data. BSD has very simple semantics for 1470 * this, no blocking and very strange errors 8) 1471 */ 1472 1473 static int tcp_recv_urg(struct sock *sk, struct msghdr *msg, int len, int flags) 1474 { 1475 struct tcp_sock *tp = tcp_sk(sk); 1476 1477 /* No URG data to read. */ 1478 if (sock_flag(sk, SOCK_URGINLINE) || !tp->urg_data || 1479 tp->urg_data == TCP_URG_READ) 1480 return -EINVAL; /* Yes this is right ! */ 1481 1482 if (sk->sk_state == TCP_CLOSE && !sock_flag(sk, SOCK_DONE)) 1483 return -ENOTCONN; 1484 1485 if (tp->urg_data & TCP_URG_VALID) { 1486 int err = 0; 1487 char c = tp->urg_data; 1488 1489 if (!(flags & MSG_PEEK)) 1490 tp->urg_data = TCP_URG_READ; 1491 1492 /* Read urgent data. */ 1493 msg->msg_flags |= MSG_OOB; 1494 1495 if (len > 0) { 1496 if (!(flags & MSG_TRUNC)) 1497 err = memcpy_to_msg(msg, &c, 1); 1498 len = 1; 1499 } else 1500 msg->msg_flags |= MSG_TRUNC; 1501 1502 return err ? -EFAULT : len; 1503 } 1504 1505 if (sk->sk_state == TCP_CLOSE || (sk->sk_shutdown & RCV_SHUTDOWN)) 1506 return 0; 1507 1508 /* Fixed the recv(..., MSG_OOB) behaviour. BSD docs and 1509 * the available implementations agree in this case: 1510 * this call should never block, independent of the 1511 * blocking state of the socket. 1512 * Mike <pall@rz.uni-karlsruhe.de> 1513 */ 1514 return -EAGAIN; 1515 } 1516 1517 static int tcp_peek_sndq(struct sock *sk, struct msghdr *msg, int len) 1518 { 1519 struct sk_buff *skb; 1520 int copied = 0, err = 0; 1521 1522 /* XXX -- need to support SO_PEEK_OFF */ 1523 1524 skb_rbtree_walk(skb, &sk->tcp_rtx_queue) { 1525 err = skb_copy_datagram_msg(skb, 0, msg, skb->len); 1526 if (err) 1527 return err; 1528 copied += skb->len; 1529 } 1530 1531 skb_queue_walk(&sk->sk_write_queue, skb) { 1532 err = skb_copy_datagram_msg(skb, 0, msg, skb->len); 1533 if (err) 1534 break; 1535 1536 copied += skb->len; 1537 } 1538 1539 return err ?: copied; 1540 } 1541 1542 /* Clean up the receive buffer for full frames taken by the user, 1543 * then send an ACK if necessary. COPIED is the number of bytes 1544 * tcp_recvmsg has given to the user so far, it speeds up the 1545 * calculation of whether or not we must ACK for the sake of 1546 * a window update. 1547 */ 1548 void tcp_cleanup_rbuf(struct sock *sk, int copied) 1549 { 1550 struct tcp_sock *tp = tcp_sk(sk); 1551 bool time_to_ack = false; 1552 1553 struct sk_buff *skb = skb_peek(&sk->sk_receive_queue); 1554 1555 WARN(skb && !before(tp->copied_seq, TCP_SKB_CB(skb)->end_seq), 1556 "cleanup rbuf bug: copied %X seq %X rcvnxt %X\n", 1557 tp->copied_seq, TCP_SKB_CB(skb)->end_seq, tp->rcv_nxt); 1558 1559 if (inet_csk_ack_scheduled(sk)) { 1560 const struct inet_connection_sock *icsk = inet_csk(sk); 1561 1562 if (/* Once-per-two-segments ACK was not sent by tcp_input.c */ 1563 tp->rcv_nxt - tp->rcv_wup > icsk->icsk_ack.rcv_mss || 1564 /* 1565 * If this read emptied read buffer, we send ACK, if 1566 * connection is not bidirectional, user drained 1567 * receive buffer and there was a small segment 1568 * in queue. 1569 */ 1570 (copied > 0 && 1571 ((icsk->icsk_ack.pending & ICSK_ACK_PUSHED2) || 1572 ((icsk->icsk_ack.pending & ICSK_ACK_PUSHED) && 1573 !inet_csk_in_pingpong_mode(sk))) && 1574 !atomic_read(&sk->sk_rmem_alloc))) 1575 time_to_ack = true; 1576 } 1577 1578 /* We send an ACK if we can now advertise a non-zero window 1579 * which has been raised "significantly". 1580 * 1581 * Even if window raised up to infinity, do not send window open ACK 1582 * in states, where we will not receive more. It is useless. 1583 */ 1584 if (copied > 0 && !time_to_ack && !(sk->sk_shutdown & RCV_SHUTDOWN)) { 1585 __u32 rcv_window_now = tcp_receive_window(tp); 1586 1587 /* Optimize, __tcp_select_window() is not cheap. */ 1588 if (2*rcv_window_now <= tp->window_clamp) { 1589 __u32 new_window = __tcp_select_window(sk); 1590 1591 /* Send ACK now, if this read freed lots of space 1592 * in our buffer. Certainly, new_window is new window. 1593 * We can advertise it now, if it is not less than current one. 1594 * "Lots" means "at least twice" here. 1595 */ 1596 if (new_window && new_window >= 2 * rcv_window_now) 1597 time_to_ack = true; 1598 } 1599 } 1600 if (time_to_ack) 1601 tcp_send_ack(sk); 1602 } 1603 1604 static struct sk_buff *tcp_recv_skb(struct sock *sk, u32 seq, u32 *off) 1605 { 1606 struct sk_buff *skb; 1607 u32 offset; 1608 1609 while ((skb = skb_peek(&sk->sk_receive_queue)) != NULL) { 1610 offset = seq - TCP_SKB_CB(skb)->seq; 1611 if (unlikely(TCP_SKB_CB(skb)->tcp_flags & TCPHDR_SYN)) { 1612 pr_err_once("%s: found a SYN, please report !\n", __func__); 1613 offset--; 1614 } 1615 if (offset < skb->len || (TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN)) { 1616 *off = offset; 1617 return skb; 1618 } 1619 /* This looks weird, but this can happen if TCP collapsing 1620 * splitted a fat GRO packet, while we released socket lock 1621 * in skb_splice_bits() 1622 */ 1623 sk_eat_skb(sk, skb); 1624 } 1625 return NULL; 1626 } 1627 1628 /* 1629 * This routine provides an alternative to tcp_recvmsg() for routines 1630 * that would like to handle copying from skbuffs directly in 'sendfile' 1631 * fashion. 1632 * Note: 1633 * - It is assumed that the socket was locked by the caller. 1634 * - The routine does not block. 1635 * - At present, there is no support for reading OOB data 1636 * or for 'peeking' the socket using this routine 1637 * (although both would be easy to implement). 1638 */ 1639 int tcp_read_sock(struct sock *sk, read_descriptor_t *desc, 1640 sk_read_actor_t recv_actor) 1641 { 1642 struct sk_buff *skb; 1643 struct tcp_sock *tp = tcp_sk(sk); 1644 u32 seq = tp->copied_seq; 1645 u32 offset; 1646 int copied = 0; 1647 1648 if (sk->sk_state == TCP_LISTEN) 1649 return -ENOTCONN; 1650 while ((skb = tcp_recv_skb(sk, seq, &offset)) != NULL) { 1651 if (offset < skb->len) { 1652 int used; 1653 size_t len; 1654 1655 len = skb->len - offset; 1656 /* Stop reading if we hit a patch of urgent data */ 1657 if (tp->urg_data) { 1658 u32 urg_offset = tp->urg_seq - seq; 1659 if (urg_offset < len) 1660 len = urg_offset; 1661 if (!len) 1662 break; 1663 } 1664 used = recv_actor(desc, skb, offset, len); 1665 if (used <= 0) { 1666 if (!copied) 1667 copied = used; 1668 break; 1669 } else if (used <= len) { 1670 seq += used; 1671 copied += used; 1672 offset += used; 1673 } 1674 /* If recv_actor drops the lock (e.g. TCP splice 1675 * receive) the skb pointer might be invalid when 1676 * getting here: tcp_collapse might have deleted it 1677 * while aggregating skbs from the socket queue. 1678 */ 1679 skb = tcp_recv_skb(sk, seq - 1, &offset); 1680 if (!skb) 1681 break; 1682 /* TCP coalescing might have appended data to the skb. 1683 * Try to splice more frags 1684 */ 1685 if (offset + 1 != skb->len) 1686 continue; 1687 } 1688 if (TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN) { 1689 sk_eat_skb(sk, skb); 1690 ++seq; 1691 break; 1692 } 1693 sk_eat_skb(sk, skb); 1694 if (!desc->count) 1695 break; 1696 WRITE_ONCE(tp->copied_seq, seq); 1697 } 1698 WRITE_ONCE(tp->copied_seq, seq); 1699 1700 tcp_rcv_space_adjust(sk); 1701 1702 /* Clean up data we have read: This will do ACK frames. */ 1703 if (copied > 0) { 1704 tcp_recv_skb(sk, seq, &offset); 1705 tcp_cleanup_rbuf(sk, copied); 1706 } 1707 return copied; 1708 } 1709 EXPORT_SYMBOL(tcp_read_sock); 1710 1711 int tcp_peek_len(struct socket *sock) 1712 { 1713 return tcp_inq(sock->sk); 1714 } 1715 EXPORT_SYMBOL(tcp_peek_len); 1716 1717 /* Make sure sk_rcvbuf is big enough to satisfy SO_RCVLOWAT hint */ 1718 int tcp_set_rcvlowat(struct sock *sk, int val) 1719 { 1720 int cap; 1721 1722 if (sk->sk_userlocks & SOCK_RCVBUF_LOCK) 1723 cap = sk->sk_rcvbuf >> 1; 1724 else 1725 cap = sock_net(sk)->ipv4.sysctl_tcp_rmem[2] >> 1; 1726 val = min(val, cap); 1727 WRITE_ONCE(sk->sk_rcvlowat, val ? : 1); 1728 1729 /* Check if we need to signal EPOLLIN right now */ 1730 tcp_data_ready(sk); 1731 1732 if (sk->sk_userlocks & SOCK_RCVBUF_LOCK) 1733 return 0; 1734 1735 val <<= 1; 1736 if (val > sk->sk_rcvbuf) { 1737 WRITE_ONCE(sk->sk_rcvbuf, val); 1738 tcp_sk(sk)->window_clamp = tcp_win_from_space(sk, val); 1739 } 1740 return 0; 1741 } 1742 EXPORT_SYMBOL(tcp_set_rcvlowat); 1743 1744 void tcp_update_recv_tstamps(struct sk_buff *skb, 1745 struct scm_timestamping_internal *tss) 1746 { 1747 if (skb->tstamp) 1748 tss->ts[0] = ktime_to_timespec64(skb->tstamp); 1749 else 1750 tss->ts[0] = (struct timespec64) {0}; 1751 1752 if (skb_hwtstamps(skb)->hwtstamp) 1753 tss->ts[2] = ktime_to_timespec64(skb_hwtstamps(skb)->hwtstamp); 1754 else 1755 tss->ts[2] = (struct timespec64) {0}; 1756 } 1757 1758 #ifdef CONFIG_MMU 1759 static const struct vm_operations_struct tcp_vm_ops = { 1760 }; 1761 1762 int tcp_mmap(struct file *file, struct socket *sock, 1763 struct vm_area_struct *vma) 1764 { 1765 if (vma->vm_flags & (VM_WRITE | VM_EXEC)) 1766 return -EPERM; 1767 vma->vm_flags &= ~(VM_MAYWRITE | VM_MAYEXEC); 1768 1769 /* Instruct vm_insert_page() to not mmap_read_lock(mm) */ 1770 vma->vm_flags |= VM_MIXEDMAP; 1771 1772 vma->vm_ops = &tcp_vm_ops; 1773 return 0; 1774 } 1775 EXPORT_SYMBOL(tcp_mmap); 1776 1777 static skb_frag_t *skb_advance_to_frag(struct sk_buff *skb, u32 offset_skb, 1778 u32 *offset_frag) 1779 { 1780 skb_frag_t *frag; 1781 1782 offset_skb -= skb_headlen(skb); 1783 if ((int)offset_skb < 0 || skb_has_frag_list(skb)) 1784 return NULL; 1785 1786 frag = skb_shinfo(skb)->frags; 1787 while (offset_skb) { 1788 if (skb_frag_size(frag) > offset_skb) { 1789 *offset_frag = offset_skb; 1790 return frag; 1791 } 1792 offset_skb -= skb_frag_size(frag); 1793 ++frag; 1794 } 1795 *offset_frag = 0; 1796 return frag; 1797 } 1798 1799 static bool can_map_frag(const skb_frag_t *frag) 1800 { 1801 return skb_frag_size(frag) == PAGE_SIZE && !skb_frag_off(frag); 1802 } 1803 1804 static int find_next_mappable_frag(const skb_frag_t *frag, 1805 int remaining_in_skb) 1806 { 1807 int offset = 0; 1808 1809 if (likely(can_map_frag(frag))) 1810 return 0; 1811 1812 while (offset < remaining_in_skb && !can_map_frag(frag)) { 1813 offset += skb_frag_size(frag); 1814 ++frag; 1815 } 1816 return offset; 1817 } 1818 1819 static void tcp_zerocopy_set_hint_for_skb(struct sock *sk, 1820 struct tcp_zerocopy_receive *zc, 1821 struct sk_buff *skb, u32 offset) 1822 { 1823 u32 frag_offset, partial_frag_remainder = 0; 1824 int mappable_offset; 1825 skb_frag_t *frag; 1826 1827 /* worst case: skip to next skb. try to improve on this case below */ 1828 zc->recv_skip_hint = skb->len - offset; 1829 1830 /* Find the frag containing this offset (and how far into that frag) */ 1831 frag = skb_advance_to_frag(skb, offset, &frag_offset); 1832 if (!frag) 1833 return; 1834 1835 if (frag_offset) { 1836 struct skb_shared_info *info = skb_shinfo(skb); 1837 1838 /* We read part of the last frag, must recvmsg() rest of skb. */ 1839 if (frag == &info->frags[info->nr_frags - 1]) 1840 return; 1841 1842 /* Else, we must at least read the remainder in this frag. */ 1843 partial_frag_remainder = skb_frag_size(frag) - frag_offset; 1844 zc->recv_skip_hint -= partial_frag_remainder; 1845 ++frag; 1846 } 1847 1848 /* partial_frag_remainder: If part way through a frag, must read rest. 1849 * mappable_offset: Bytes till next mappable frag, *not* counting bytes 1850 * in partial_frag_remainder. 1851 */ 1852 mappable_offset = find_next_mappable_frag(frag, zc->recv_skip_hint); 1853 zc->recv_skip_hint = mappable_offset + partial_frag_remainder; 1854 } 1855 1856 static int tcp_recvmsg_locked(struct sock *sk, struct msghdr *msg, size_t len, 1857 int nonblock, int flags, 1858 struct scm_timestamping_internal *tss, 1859 int *cmsg_flags); 1860 static int receive_fallback_to_copy(struct sock *sk, 1861 struct tcp_zerocopy_receive *zc, int inq, 1862 struct scm_timestamping_internal *tss) 1863 { 1864 unsigned long copy_address = (unsigned long)zc->copybuf_address; 1865 struct msghdr msg = {}; 1866 struct iovec iov; 1867 int err; 1868 1869 zc->length = 0; 1870 zc->recv_skip_hint = 0; 1871 1872 if (copy_address != zc->copybuf_address) 1873 return -EINVAL; 1874 1875 err = import_single_range(READ, (void __user *)copy_address, 1876 inq, &iov, &msg.msg_iter); 1877 if (err) 1878 return err; 1879 1880 err = tcp_recvmsg_locked(sk, &msg, inq, /*nonblock=*/1, /*flags=*/0, 1881 tss, &zc->msg_flags); 1882 if (err < 0) 1883 return err; 1884 1885 zc->copybuf_len = err; 1886 if (likely(zc->copybuf_len)) { 1887 struct sk_buff *skb; 1888 u32 offset; 1889 1890 skb = tcp_recv_skb(sk, tcp_sk(sk)->copied_seq, &offset); 1891 if (skb) 1892 tcp_zerocopy_set_hint_for_skb(sk, zc, skb, offset); 1893 } 1894 return 0; 1895 } 1896 1897 static int tcp_copy_straggler_data(struct tcp_zerocopy_receive *zc, 1898 struct sk_buff *skb, u32 copylen, 1899 u32 *offset, u32 *seq) 1900 { 1901 unsigned long copy_address = (unsigned long)zc->copybuf_address; 1902 struct msghdr msg = {}; 1903 struct iovec iov; 1904 int err; 1905 1906 if (copy_address != zc->copybuf_address) 1907 return -EINVAL; 1908 1909 err = import_single_range(READ, (void __user *)copy_address, 1910 copylen, &iov, &msg.msg_iter); 1911 if (err) 1912 return err; 1913 err = skb_copy_datagram_msg(skb, *offset, &msg, copylen); 1914 if (err) 1915 return err; 1916 zc->recv_skip_hint -= copylen; 1917 *offset += copylen; 1918 *seq += copylen; 1919 return (__s32)copylen; 1920 } 1921 1922 static int tcp_zc_handle_leftover(struct tcp_zerocopy_receive *zc, 1923 struct sock *sk, 1924 struct sk_buff *skb, 1925 u32 *seq, 1926 s32 copybuf_len, 1927 struct scm_timestamping_internal *tss) 1928 { 1929 u32 offset, copylen = min_t(u32, copybuf_len, zc->recv_skip_hint); 1930 1931 if (!copylen) 1932 return 0; 1933 /* skb is null if inq < PAGE_SIZE. */ 1934 if (skb) { 1935 offset = *seq - TCP_SKB_CB(skb)->seq; 1936 } else { 1937 skb = tcp_recv_skb(sk, *seq, &offset); 1938 if (TCP_SKB_CB(skb)->has_rxtstamp) { 1939 tcp_update_recv_tstamps(skb, tss); 1940 zc->msg_flags |= TCP_CMSG_TS; 1941 } 1942 } 1943 1944 zc->copybuf_len = tcp_copy_straggler_data(zc, skb, copylen, &offset, 1945 seq); 1946 return zc->copybuf_len < 0 ? 0 : copylen; 1947 } 1948 1949 static int tcp_zerocopy_vm_insert_batch_error(struct vm_area_struct *vma, 1950 struct page **pending_pages, 1951 unsigned long pages_remaining, 1952 unsigned long *address, 1953 u32 *length, 1954 u32 *seq, 1955 struct tcp_zerocopy_receive *zc, 1956 u32 total_bytes_to_map, 1957 int err) 1958 { 1959 /* At least one page did not map. Try zapping if we skipped earlier. */ 1960 if (err == -EBUSY && 1961 zc->flags & TCP_RECEIVE_ZEROCOPY_FLAG_TLB_CLEAN_HINT) { 1962 u32 maybe_zap_len; 1963 1964 maybe_zap_len = total_bytes_to_map - /* All bytes to map */ 1965 *length + /* Mapped or pending */ 1966 (pages_remaining * PAGE_SIZE); /* Failed map. */ 1967 zap_page_range(vma, *address, maybe_zap_len); 1968 err = 0; 1969 } 1970 1971 if (!err) { 1972 unsigned long leftover_pages = pages_remaining; 1973 int bytes_mapped; 1974 1975 /* We called zap_page_range, try to reinsert. */ 1976 err = vm_insert_pages(vma, *address, 1977 pending_pages, 1978 &pages_remaining); 1979 bytes_mapped = PAGE_SIZE * (leftover_pages - pages_remaining); 1980 *seq += bytes_mapped; 1981 *address += bytes_mapped; 1982 } 1983 if (err) { 1984 /* Either we were unable to zap, OR we zapped, retried an 1985 * insert, and still had an issue. Either ways, pages_remaining 1986 * is the number of pages we were unable to map, and we unroll 1987 * some state we speculatively touched before. 1988 */ 1989 const int bytes_not_mapped = PAGE_SIZE * pages_remaining; 1990 1991 *length -= bytes_not_mapped; 1992 zc->recv_skip_hint += bytes_not_mapped; 1993 } 1994 return err; 1995 } 1996 1997 static int tcp_zerocopy_vm_insert_batch(struct vm_area_struct *vma, 1998 struct page **pages, 1999 unsigned int pages_to_map, 2000 unsigned long *address, 2001 u32 *length, 2002 u32 *seq, 2003 struct tcp_zerocopy_receive *zc, 2004 u32 total_bytes_to_map) 2005 { 2006 unsigned long pages_remaining = pages_to_map; 2007 unsigned int pages_mapped; 2008 unsigned int bytes_mapped; 2009 int err; 2010 2011 err = vm_insert_pages(vma, *address, pages, &pages_remaining); 2012 pages_mapped = pages_to_map - (unsigned int)pages_remaining; 2013 bytes_mapped = PAGE_SIZE * pages_mapped; 2014 /* Even if vm_insert_pages fails, it may have partially succeeded in 2015 * mapping (some but not all of the pages). 2016 */ 2017 *seq += bytes_mapped; 2018 *address += bytes_mapped; 2019 2020 if (likely(!err)) 2021 return 0; 2022 2023 /* Error: maybe zap and retry + rollback state for failed inserts. */ 2024 return tcp_zerocopy_vm_insert_batch_error(vma, pages + pages_mapped, 2025 pages_remaining, address, length, seq, zc, total_bytes_to_map, 2026 err); 2027 } 2028 2029 #define TCP_VALID_ZC_MSG_FLAGS (TCP_CMSG_TS) 2030 static void tcp_zc_finalize_rx_tstamp(struct sock *sk, 2031 struct tcp_zerocopy_receive *zc, 2032 struct scm_timestamping_internal *tss) 2033 { 2034 unsigned long msg_control_addr; 2035 struct msghdr cmsg_dummy; 2036 2037 msg_control_addr = (unsigned long)zc->msg_control; 2038 cmsg_dummy.msg_control = (void *)msg_control_addr; 2039 cmsg_dummy.msg_controllen = 2040 (__kernel_size_t)zc->msg_controllen; 2041 cmsg_dummy.msg_flags = in_compat_syscall() 2042 ? MSG_CMSG_COMPAT : 0; 2043 cmsg_dummy.msg_control_is_user = true; 2044 zc->msg_flags = 0; 2045 if (zc->msg_control == msg_control_addr && 2046 zc->msg_controllen == cmsg_dummy.msg_controllen) { 2047 tcp_recv_timestamp(&cmsg_dummy, sk, tss); 2048 zc->msg_control = (__u64) 2049 ((uintptr_t)cmsg_dummy.msg_control); 2050 zc->msg_controllen = 2051 (__u64)cmsg_dummy.msg_controllen; 2052 zc->msg_flags = (__u32)cmsg_dummy.msg_flags; 2053 } 2054 } 2055 2056 #define TCP_ZEROCOPY_PAGE_BATCH_SIZE 32 2057 static int tcp_zerocopy_receive(struct sock *sk, 2058 struct tcp_zerocopy_receive *zc, 2059 struct scm_timestamping_internal *tss) 2060 { 2061 u32 length = 0, offset, vma_len, avail_len, copylen = 0; 2062 unsigned long address = (unsigned long)zc->address; 2063 struct page *pages[TCP_ZEROCOPY_PAGE_BATCH_SIZE]; 2064 s32 copybuf_len = zc->copybuf_len; 2065 struct tcp_sock *tp = tcp_sk(sk); 2066 const skb_frag_t *frags = NULL; 2067 unsigned int pages_to_map = 0; 2068 struct vm_area_struct *vma; 2069 struct sk_buff *skb = NULL; 2070 u32 seq = tp->copied_seq; 2071 u32 total_bytes_to_map; 2072 int inq = tcp_inq(sk); 2073 int ret; 2074 2075 zc->copybuf_len = 0; 2076 zc->msg_flags = 0; 2077 2078 if (address & (PAGE_SIZE - 1) || address != zc->address) 2079 return -EINVAL; 2080 2081 if (sk->sk_state == TCP_LISTEN) 2082 return -ENOTCONN; 2083 2084 sock_rps_record_flow(sk); 2085 2086 if (inq && inq <= copybuf_len) 2087 return receive_fallback_to_copy(sk, zc, inq, tss); 2088 2089 if (inq < PAGE_SIZE) { 2090 zc->length = 0; 2091 zc->recv_skip_hint = inq; 2092 if (!inq && sock_flag(sk, SOCK_DONE)) 2093 return -EIO; 2094 return 0; 2095 } 2096 2097 mmap_read_lock(current->mm); 2098 2099 vma = vma_lookup(current->mm, address); 2100 if (!vma || vma->vm_ops != &tcp_vm_ops) { 2101 mmap_read_unlock(current->mm); 2102 return -EINVAL; 2103 } 2104 vma_len = min_t(unsigned long, zc->length, vma->vm_end - address); 2105 avail_len = min_t(u32, vma_len, inq); 2106 total_bytes_to_map = avail_len & ~(PAGE_SIZE - 1); 2107 if (total_bytes_to_map) { 2108 if (!(zc->flags & TCP_RECEIVE_ZEROCOPY_FLAG_TLB_CLEAN_HINT)) 2109 zap_page_range(vma, address, total_bytes_to_map); 2110 zc->length = total_bytes_to_map; 2111 zc->recv_skip_hint = 0; 2112 } else { 2113 zc->length = avail_len; 2114 zc->recv_skip_hint = avail_len; 2115 } 2116 ret = 0; 2117 while (length + PAGE_SIZE <= zc->length) { 2118 int mappable_offset; 2119 struct page *page; 2120 2121 if (zc->recv_skip_hint < PAGE_SIZE) { 2122 u32 offset_frag; 2123 2124 if (skb) { 2125 if (zc->recv_skip_hint > 0) 2126 break; 2127 skb = skb->next; 2128 offset = seq - TCP_SKB_CB(skb)->seq; 2129 } else { 2130 skb = tcp_recv_skb(sk, seq, &offset); 2131 } 2132 2133 if (TCP_SKB_CB(skb)->has_rxtstamp) { 2134 tcp_update_recv_tstamps(skb, tss); 2135 zc->msg_flags |= TCP_CMSG_TS; 2136 } 2137 zc->recv_skip_hint = skb->len - offset; 2138 frags = skb_advance_to_frag(skb, offset, &offset_frag); 2139 if (!frags || offset_frag) 2140 break; 2141 } 2142 2143 mappable_offset = find_next_mappable_frag(frags, 2144 zc->recv_skip_hint); 2145 if (mappable_offset) { 2146 zc->recv_skip_hint = mappable_offset; 2147 break; 2148 } 2149 page = skb_frag_page(frags); 2150 prefetchw(page); 2151 pages[pages_to_map++] = page; 2152 length += PAGE_SIZE; 2153 zc->recv_skip_hint -= PAGE_SIZE; 2154 frags++; 2155 if (pages_to_map == TCP_ZEROCOPY_PAGE_BATCH_SIZE || 2156 zc->recv_skip_hint < PAGE_SIZE) { 2157 /* Either full batch, or we're about to go to next skb 2158 * (and we cannot unroll failed ops across skbs). 2159 */ 2160 ret = tcp_zerocopy_vm_insert_batch(vma, pages, 2161 pages_to_map, 2162 &address, &length, 2163 &seq, zc, 2164 total_bytes_to_map); 2165 if (ret) 2166 goto out; 2167 pages_to_map = 0; 2168 } 2169 } 2170 if (pages_to_map) { 2171 ret = tcp_zerocopy_vm_insert_batch(vma, pages, pages_to_map, 2172 &address, &length, &seq, 2173 zc, total_bytes_to_map); 2174 } 2175 out: 2176 mmap_read_unlock(current->mm); 2177 /* Try to copy straggler data. */ 2178 if (!ret) 2179 copylen = tcp_zc_handle_leftover(zc, sk, skb, &seq, copybuf_len, tss); 2180 2181 if (length + copylen) { 2182 WRITE_ONCE(tp->copied_seq, seq); 2183 tcp_rcv_space_adjust(sk); 2184 2185 /* Clean up data we have read: This will do ACK frames. */ 2186 tcp_recv_skb(sk, seq, &offset); 2187 tcp_cleanup_rbuf(sk, length + copylen); 2188 ret = 0; 2189 if (length == zc->length) 2190 zc->recv_skip_hint = 0; 2191 } else { 2192 if (!zc->recv_skip_hint && sock_flag(sk, SOCK_DONE)) 2193 ret = -EIO; 2194 } 2195 zc->length = length; 2196 return ret; 2197 } 2198 #endif 2199 2200 /* Similar to __sock_recv_timestamp, but does not require an skb */ 2201 void tcp_recv_timestamp(struct msghdr *msg, const struct sock *sk, 2202 struct scm_timestamping_internal *tss) 2203 { 2204 int new_tstamp = sock_flag(sk, SOCK_TSTAMP_NEW); 2205 bool has_timestamping = false; 2206 2207 if (tss->ts[0].tv_sec || tss->ts[0].tv_nsec) { 2208 if (sock_flag(sk, SOCK_RCVTSTAMP)) { 2209 if (sock_flag(sk, SOCK_RCVTSTAMPNS)) { 2210 if (new_tstamp) { 2211 struct __kernel_timespec kts = { 2212 .tv_sec = tss->ts[0].tv_sec, 2213 .tv_nsec = tss->ts[0].tv_nsec, 2214 }; 2215 put_cmsg(msg, SOL_SOCKET, SO_TIMESTAMPNS_NEW, 2216 sizeof(kts), &kts); 2217 } else { 2218 struct __kernel_old_timespec ts_old = { 2219 .tv_sec = tss->ts[0].tv_sec, 2220 .tv_nsec = tss->ts[0].tv_nsec, 2221 }; 2222 put_cmsg(msg, SOL_SOCKET, SO_TIMESTAMPNS_OLD, 2223 sizeof(ts_old), &ts_old); 2224 } 2225 } else { 2226 if (new_tstamp) { 2227 struct __kernel_sock_timeval stv = { 2228 .tv_sec = tss->ts[0].tv_sec, 2229 .tv_usec = tss->ts[0].tv_nsec / 1000, 2230 }; 2231 put_cmsg(msg, SOL_SOCKET, SO_TIMESTAMP_NEW, 2232 sizeof(stv), &stv); 2233 } else { 2234 struct __kernel_old_timeval tv = { 2235 .tv_sec = tss->ts[0].tv_sec, 2236 .tv_usec = tss->ts[0].tv_nsec / 1000, 2237 }; 2238 put_cmsg(msg, SOL_SOCKET, SO_TIMESTAMP_OLD, 2239 sizeof(tv), &tv); 2240 } 2241 } 2242 } 2243 2244 if (sk->sk_tsflags & SOF_TIMESTAMPING_SOFTWARE) 2245 has_timestamping = true; 2246 else 2247 tss->ts[0] = (struct timespec64) {0}; 2248 } 2249 2250 if (tss->ts[2].tv_sec || tss->ts[2].tv_nsec) { 2251 if (sk->sk_tsflags & SOF_TIMESTAMPING_RAW_HARDWARE) 2252 has_timestamping = true; 2253 else 2254 tss->ts[2] = (struct timespec64) {0}; 2255 } 2256 2257 if (has_timestamping) { 2258 tss->ts[1] = (struct timespec64) {0}; 2259 if (sock_flag(sk, SOCK_TSTAMP_NEW)) 2260 put_cmsg_scm_timestamping64(msg, tss); 2261 else 2262 put_cmsg_scm_timestamping(msg, tss); 2263 } 2264 } 2265 2266 static int tcp_inq_hint(struct sock *sk) 2267 { 2268 const struct tcp_sock *tp = tcp_sk(sk); 2269 u32 copied_seq = READ_ONCE(tp->copied_seq); 2270 u32 rcv_nxt = READ_ONCE(tp->rcv_nxt); 2271 int inq; 2272 2273 inq = rcv_nxt - copied_seq; 2274 if (unlikely(inq < 0 || copied_seq != READ_ONCE(tp->copied_seq))) { 2275 lock_sock(sk); 2276 inq = tp->rcv_nxt - tp->copied_seq; 2277 release_sock(sk); 2278 } 2279 /* After receiving a FIN, tell the user-space to continue reading 2280 * by returning a non-zero inq. 2281 */ 2282 if (inq == 0 && sock_flag(sk, SOCK_DONE)) 2283 inq = 1; 2284 return inq; 2285 } 2286 2287 /* 2288 * This routine copies from a sock struct into the user buffer. 2289 * 2290 * Technical note: in 2.3 we work on _locked_ socket, so that 2291 * tricks with *seq access order and skb->users are not required. 2292 * Probably, code can be easily improved even more. 2293 */ 2294 2295 static int tcp_recvmsg_locked(struct sock *sk, struct msghdr *msg, size_t len, 2296 int nonblock, int flags, 2297 struct scm_timestamping_internal *tss, 2298 int *cmsg_flags) 2299 { 2300 struct tcp_sock *tp = tcp_sk(sk); 2301 int copied = 0; 2302 u32 peek_seq; 2303 u32 *seq; 2304 unsigned long used; 2305 int err; 2306 int target; /* Read at least this many bytes */ 2307 long timeo; 2308 struct sk_buff *skb, *last; 2309 u32 urg_hole = 0; 2310 2311 err = -ENOTCONN; 2312 if (sk->sk_state == TCP_LISTEN) 2313 goto out; 2314 2315 if (tp->recvmsg_inq) 2316 *cmsg_flags = TCP_CMSG_INQ; 2317 timeo = sock_rcvtimeo(sk, nonblock); 2318 2319 /* Urgent data needs to be handled specially. */ 2320 if (flags & MSG_OOB) 2321 goto recv_urg; 2322 2323 if (unlikely(tp->repair)) { 2324 err = -EPERM; 2325 if (!(flags & MSG_PEEK)) 2326 goto out; 2327 2328 if (tp->repair_queue == TCP_SEND_QUEUE) 2329 goto recv_sndq; 2330 2331 err = -EINVAL; 2332 if (tp->repair_queue == TCP_NO_QUEUE) 2333 goto out; 2334 2335 /* 'common' recv queue MSG_PEEK-ing */ 2336 } 2337 2338 seq = &tp->copied_seq; 2339 if (flags & MSG_PEEK) { 2340 peek_seq = tp->copied_seq; 2341 seq = &peek_seq; 2342 } 2343 2344 target = sock_rcvlowat(sk, flags & MSG_WAITALL, len); 2345 2346 do { 2347 u32 offset; 2348 2349 /* Are we at urgent data? Stop if we have read anything or have SIGURG pending. */ 2350 if (tp->urg_data && tp->urg_seq == *seq) { 2351 if (copied) 2352 break; 2353 if (signal_pending(current)) { 2354 copied = timeo ? sock_intr_errno(timeo) : -EAGAIN; 2355 break; 2356 } 2357 } 2358 2359 /* Next get a buffer. */ 2360 2361 last = skb_peek_tail(&sk->sk_receive_queue); 2362 skb_queue_walk(&sk->sk_receive_queue, skb) { 2363 last = skb; 2364 /* Now that we have two receive queues this 2365 * shouldn't happen. 2366 */ 2367 if (WARN(before(*seq, TCP_SKB_CB(skb)->seq), 2368 "TCP recvmsg seq # bug: copied %X, seq %X, rcvnxt %X, fl %X\n", 2369 *seq, TCP_SKB_CB(skb)->seq, tp->rcv_nxt, 2370 flags)) 2371 break; 2372 2373 offset = *seq - TCP_SKB_CB(skb)->seq; 2374 if (unlikely(TCP_SKB_CB(skb)->tcp_flags & TCPHDR_SYN)) { 2375 pr_err_once("%s: found a SYN, please report !\n", __func__); 2376 offset--; 2377 } 2378 if (offset < skb->len) 2379 goto found_ok_skb; 2380 if (TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN) 2381 goto found_fin_ok; 2382 WARN(!(flags & MSG_PEEK), 2383 "TCP recvmsg seq # bug 2: copied %X, seq %X, rcvnxt %X, fl %X\n", 2384 *seq, TCP_SKB_CB(skb)->seq, tp->rcv_nxt, flags); 2385 } 2386 2387 /* Well, if we have backlog, try to process it now yet. */ 2388 2389 if (copied >= target && !READ_ONCE(sk->sk_backlog.tail)) 2390 break; 2391 2392 if (copied) { 2393 if (sk->sk_err || 2394 sk->sk_state == TCP_CLOSE || 2395 (sk->sk_shutdown & RCV_SHUTDOWN) || 2396 !timeo || 2397 signal_pending(current)) 2398 break; 2399 } else { 2400 if (sock_flag(sk, SOCK_DONE)) 2401 break; 2402 2403 if (sk->sk_err) { 2404 copied = sock_error(sk); 2405 break; 2406 } 2407 2408 if (sk->sk_shutdown & RCV_SHUTDOWN) 2409 break; 2410 2411 if (sk->sk_state == TCP_CLOSE) { 2412 /* This occurs when user tries to read 2413 * from never connected socket. 2414 */ 2415 copied = -ENOTCONN; 2416 break; 2417 } 2418 2419 if (!timeo) { 2420 copied = -EAGAIN; 2421 break; 2422 } 2423 2424 if (signal_pending(current)) { 2425 copied = sock_intr_errno(timeo); 2426 break; 2427 } 2428 } 2429 2430 tcp_cleanup_rbuf(sk, copied); 2431 2432 if (copied >= target) { 2433 /* Do not sleep, just process backlog. */ 2434 release_sock(sk); 2435 lock_sock(sk); 2436 } else { 2437 sk_wait_data(sk, &timeo, last); 2438 } 2439 2440 if ((flags & MSG_PEEK) && 2441 (peek_seq - copied - urg_hole != tp->copied_seq)) { 2442 net_dbg_ratelimited("TCP(%s:%d): Application bug, race in MSG_PEEK\n", 2443 current->comm, 2444 task_pid_nr(current)); 2445 peek_seq = tp->copied_seq; 2446 } 2447 continue; 2448 2449 found_ok_skb: 2450 /* Ok so how much can we use? */ 2451 used = skb->len - offset; 2452 if (len < used) 2453 used = len; 2454 2455 /* Do we have urgent data here? */ 2456 if (tp->urg_data) { 2457 u32 urg_offset = tp->urg_seq - *seq; 2458 if (urg_offset < used) { 2459 if (!urg_offset) { 2460 if (!sock_flag(sk, SOCK_URGINLINE)) { 2461 WRITE_ONCE(*seq, *seq + 1); 2462 urg_hole++; 2463 offset++; 2464 used--; 2465 if (!used) 2466 goto skip_copy; 2467 } 2468 } else 2469 used = urg_offset; 2470 } 2471 } 2472 2473 if (!(flags & MSG_TRUNC)) { 2474 err = skb_copy_datagram_msg(skb, offset, msg, used); 2475 if (err) { 2476 /* Exception. Bailout! */ 2477 if (!copied) 2478 copied = -EFAULT; 2479 break; 2480 } 2481 } 2482 2483 WRITE_ONCE(*seq, *seq + used); 2484 copied += used; 2485 len -= used; 2486 2487 tcp_rcv_space_adjust(sk); 2488 2489 skip_copy: 2490 if (tp->urg_data && after(tp->copied_seq, tp->urg_seq)) { 2491 tp->urg_data = 0; 2492 tcp_fast_path_check(sk); 2493 } 2494 2495 if (TCP_SKB_CB(skb)->has_rxtstamp) { 2496 tcp_update_recv_tstamps(skb, tss); 2497 *cmsg_flags |= TCP_CMSG_TS; 2498 } 2499 2500 if (used + offset < skb->len) 2501 continue; 2502 2503 if (TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN) 2504 goto found_fin_ok; 2505 if (!(flags & MSG_PEEK)) 2506 sk_eat_skb(sk, skb); 2507 continue; 2508 2509 found_fin_ok: 2510 /* Process the FIN. */ 2511 WRITE_ONCE(*seq, *seq + 1); 2512 if (!(flags & MSG_PEEK)) 2513 sk_eat_skb(sk, skb); 2514 break; 2515 } while (len > 0); 2516 2517 /* According to UNIX98, msg_name/msg_namelen are ignored 2518 * on connected socket. I was just happy when found this 8) --ANK 2519 */ 2520 2521 /* Clean up data we have read: This will do ACK frames. */ 2522 tcp_cleanup_rbuf(sk, copied); 2523 return copied; 2524 2525 out: 2526 return err; 2527 2528 recv_urg: 2529 err = tcp_recv_urg(sk, msg, len, flags); 2530 goto out; 2531 2532 recv_sndq: 2533 err = tcp_peek_sndq(sk, msg, len); 2534 goto out; 2535 } 2536 2537 int tcp_recvmsg(struct sock *sk, struct msghdr *msg, size_t len, int nonblock, 2538 int flags, int *addr_len) 2539 { 2540 int cmsg_flags = 0, ret, inq; 2541 struct scm_timestamping_internal tss; 2542 2543 if (unlikely(flags & MSG_ERRQUEUE)) 2544 return inet_recv_error(sk, msg, len, addr_len); 2545 2546 if (sk_can_busy_loop(sk) && 2547 skb_queue_empty_lockless(&sk->sk_receive_queue) && 2548 sk->sk_state == TCP_ESTABLISHED) 2549 sk_busy_loop(sk, nonblock); 2550 2551 lock_sock(sk); 2552 ret = tcp_recvmsg_locked(sk, msg, len, nonblock, flags, &tss, 2553 &cmsg_flags); 2554 release_sock(sk); 2555 2556 if (cmsg_flags && ret >= 0) { 2557 if (cmsg_flags & TCP_CMSG_TS) 2558 tcp_recv_timestamp(msg, sk, &tss); 2559 if (cmsg_flags & TCP_CMSG_INQ) { 2560 inq = tcp_inq_hint(sk); 2561 put_cmsg(msg, SOL_TCP, TCP_CM_INQ, sizeof(inq), &inq); 2562 } 2563 } 2564 return ret; 2565 } 2566 EXPORT_SYMBOL(tcp_recvmsg); 2567 2568 void tcp_set_state(struct sock *sk, int state) 2569 { 2570 int oldstate = sk->sk_state; 2571 2572 /* We defined a new enum for TCP states that are exported in BPF 2573 * so as not force the internal TCP states to be frozen. The 2574 * following checks will detect if an internal state value ever 2575 * differs from the BPF value. If this ever happens, then we will 2576 * need to remap the internal value to the BPF value before calling 2577 * tcp_call_bpf_2arg. 2578 */ 2579 BUILD_BUG_ON((int)BPF_TCP_ESTABLISHED != (int)TCP_ESTABLISHED); 2580 BUILD_BUG_ON((int)BPF_TCP_SYN_SENT != (int)TCP_SYN_SENT); 2581 BUILD_BUG_ON((int)BPF_TCP_SYN_RECV != (int)TCP_SYN_RECV); 2582 BUILD_BUG_ON((int)BPF_TCP_FIN_WAIT1 != (int)TCP_FIN_WAIT1); 2583 BUILD_BUG_ON((int)BPF_TCP_FIN_WAIT2 != (int)TCP_FIN_WAIT2); 2584 BUILD_BUG_ON((int)BPF_TCP_TIME_WAIT != (int)TCP_TIME_WAIT); 2585 BUILD_BUG_ON((int)BPF_TCP_CLOSE != (int)TCP_CLOSE); 2586 BUILD_BUG_ON((int)BPF_TCP_CLOSE_WAIT != (int)TCP_CLOSE_WAIT); 2587 BUILD_BUG_ON((int)BPF_TCP_LAST_ACK != (int)TCP_LAST_ACK); 2588 BUILD_BUG_ON((int)BPF_TCP_LISTEN != (int)TCP_LISTEN); 2589 BUILD_BUG_ON((int)BPF_TCP_CLOSING != (int)TCP_CLOSING); 2590 BUILD_BUG_ON((int)BPF_TCP_NEW_SYN_RECV != (int)TCP_NEW_SYN_RECV); 2591 BUILD_BUG_ON((int)BPF_TCP_MAX_STATES != (int)TCP_MAX_STATES); 2592 2593 /* bpf uapi header bpf.h defines an anonymous enum with values 2594 * BPF_TCP_* used by bpf programs. Currently gcc built vmlinux 2595 * is able to emit this enum in DWARF due to the above BUILD_BUG_ON. 2596 * But clang built vmlinux does not have this enum in DWARF 2597 * since clang removes the above code before generating IR/debuginfo. 2598 * Let us explicitly emit the type debuginfo to ensure the 2599 * above-mentioned anonymous enum in the vmlinux DWARF and hence BTF 2600 * regardless of which compiler is used. 2601 */ 2602 BTF_TYPE_EMIT_ENUM(BPF_TCP_ESTABLISHED); 2603 2604 if (BPF_SOCK_OPS_TEST_FLAG(tcp_sk(sk), BPF_SOCK_OPS_STATE_CB_FLAG)) 2605 tcp_call_bpf_2arg(sk, BPF_SOCK_OPS_STATE_CB, oldstate, state); 2606 2607 switch (state) { 2608 case TCP_ESTABLISHED: 2609 if (oldstate != TCP_ESTABLISHED) 2610 TCP_INC_STATS(sock_net(sk), TCP_MIB_CURRESTAB); 2611 break; 2612 2613 case TCP_CLOSE: 2614 if (oldstate == TCP_CLOSE_WAIT || oldstate == TCP_ESTABLISHED) 2615 TCP_INC_STATS(sock_net(sk), TCP_MIB_ESTABRESETS); 2616 2617 sk->sk_prot->unhash(sk); 2618 if (inet_csk(sk)->icsk_bind_hash && 2619 !(sk->sk_userlocks & SOCK_BINDPORT_LOCK)) 2620 inet_put_port(sk); 2621 fallthrough; 2622 default: 2623 if (oldstate == TCP_ESTABLISHED) 2624 TCP_DEC_STATS(sock_net(sk), TCP_MIB_CURRESTAB); 2625 } 2626 2627 /* Change state AFTER socket is unhashed to avoid closed 2628 * socket sitting in hash tables. 2629 */ 2630 inet_sk_state_store(sk, state); 2631 } 2632 EXPORT_SYMBOL_GPL(tcp_set_state); 2633 2634 /* 2635 * State processing on a close. This implements the state shift for 2636 * sending our FIN frame. Note that we only send a FIN for some 2637 * states. A shutdown() may have already sent the FIN, or we may be 2638 * closed. 2639 */ 2640 2641 static const unsigned char new_state[16] = { 2642 /* current state: new state: action: */ 2643 [0 /* (Invalid) */] = TCP_CLOSE, 2644 [TCP_ESTABLISHED] = TCP_FIN_WAIT1 | TCP_ACTION_FIN, 2645 [TCP_SYN_SENT] = TCP_CLOSE, 2646 [TCP_SYN_RECV] = TCP_FIN_WAIT1 | TCP_ACTION_FIN, 2647 [TCP_FIN_WAIT1] = TCP_FIN_WAIT1, 2648 [TCP_FIN_WAIT2] = TCP_FIN_WAIT2, 2649 [TCP_TIME_WAIT] = TCP_CLOSE, 2650 [TCP_CLOSE] = TCP_CLOSE, 2651 [TCP_CLOSE_WAIT] = TCP_LAST_ACK | TCP_ACTION_FIN, 2652 [TCP_LAST_ACK] = TCP_LAST_ACK, 2653 [TCP_LISTEN] = TCP_CLOSE, 2654 [TCP_CLOSING] = TCP_CLOSING, 2655 [TCP_NEW_SYN_RECV] = TCP_CLOSE, /* should not happen ! */ 2656 }; 2657 2658 static int tcp_close_state(struct sock *sk) 2659 { 2660 int next = (int)new_state[sk->sk_state]; 2661 int ns = next & TCP_STATE_MASK; 2662 2663 tcp_set_state(sk, ns); 2664 2665 return next & TCP_ACTION_FIN; 2666 } 2667 2668 /* 2669 * Shutdown the sending side of a connection. Much like close except 2670 * that we don't receive shut down or sock_set_flag(sk, SOCK_DEAD). 2671 */ 2672 2673 void tcp_shutdown(struct sock *sk, int how) 2674 { 2675 /* We need to grab some memory, and put together a FIN, 2676 * and then put it into the queue to be sent. 2677 * Tim MacKenzie(tym@dibbler.cs.monash.edu.au) 4 Dec '92. 2678 */ 2679 if (!(how & SEND_SHUTDOWN)) 2680 return; 2681 2682 /* If we've already sent a FIN, or it's a closed state, skip this. */ 2683 if ((1 << sk->sk_state) & 2684 (TCPF_ESTABLISHED | TCPF_SYN_SENT | 2685 TCPF_SYN_RECV | TCPF_CLOSE_WAIT)) { 2686 /* Clear out any half completed packets. FIN if needed. */ 2687 if (tcp_close_state(sk)) 2688 tcp_send_fin(sk); 2689 } 2690 } 2691 EXPORT_SYMBOL(tcp_shutdown); 2692 2693 bool tcp_check_oom(struct sock *sk, int shift) 2694 { 2695 bool too_many_orphans, out_of_socket_memory; 2696 2697 too_many_orphans = tcp_too_many_orphans(sk, shift); 2698 out_of_socket_memory = tcp_out_of_memory(sk); 2699 2700 if (too_many_orphans) 2701 net_info_ratelimited("too many orphaned sockets\n"); 2702 if (out_of_socket_memory) 2703 net_info_ratelimited("out of memory -- consider tuning tcp_mem\n"); 2704 return too_many_orphans || out_of_socket_memory; 2705 } 2706 2707 void __tcp_close(struct sock *sk, long timeout) 2708 { 2709 struct sk_buff *skb; 2710 int data_was_unread = 0; 2711 int state; 2712 2713 sk->sk_shutdown = SHUTDOWN_MASK; 2714 2715 if (sk->sk_state == TCP_LISTEN) { 2716 tcp_set_state(sk, TCP_CLOSE); 2717 2718 /* Special case. */ 2719 inet_csk_listen_stop(sk); 2720 2721 goto adjudge_to_death; 2722 } 2723 2724 /* We need to flush the recv. buffs. We do this only on the 2725 * descriptor close, not protocol-sourced closes, because the 2726 * reader process may not have drained the data yet! 2727 */ 2728 while ((skb = __skb_dequeue(&sk->sk_receive_queue)) != NULL) { 2729 u32 len = TCP_SKB_CB(skb)->end_seq - TCP_SKB_CB(skb)->seq; 2730 2731 if (TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN) 2732 len--; 2733 data_was_unread += len; 2734 __kfree_skb(skb); 2735 } 2736 2737 sk_mem_reclaim(sk); 2738 2739 /* If socket has been already reset (e.g. in tcp_reset()) - kill it. */ 2740 if (sk->sk_state == TCP_CLOSE) 2741 goto adjudge_to_death; 2742 2743 /* As outlined in RFC 2525, section 2.17, we send a RST here because 2744 * data was lost. To witness the awful effects of the old behavior of 2745 * always doing a FIN, run an older 2.1.x kernel or 2.0.x, start a bulk 2746 * GET in an FTP client, suspend the process, wait for the client to 2747 * advertise a zero window, then kill -9 the FTP client, wheee... 2748 * Note: timeout is always zero in such a case. 2749 */ 2750 if (unlikely(tcp_sk(sk)->repair)) { 2751 sk->sk_prot->disconnect(sk, 0); 2752 } else if (data_was_unread) { 2753 /* Unread data was tossed, zap the connection. */ 2754 NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPABORTONCLOSE); 2755 tcp_set_state(sk, TCP_CLOSE); 2756 tcp_send_active_reset(sk, sk->sk_allocation); 2757 } else if (sock_flag(sk, SOCK_LINGER) && !sk->sk_lingertime) { 2758 /* Check zero linger _after_ checking for unread data. */ 2759 sk->sk_prot->disconnect(sk, 0); 2760 NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPABORTONDATA); 2761 } else if (tcp_close_state(sk)) { 2762 /* We FIN if the application ate all the data before 2763 * zapping the connection. 2764 */ 2765 2766 /* RED-PEN. Formally speaking, we have broken TCP state 2767 * machine. State transitions: 2768 * 2769 * TCP_ESTABLISHED -> TCP_FIN_WAIT1 2770 * TCP_SYN_RECV -> TCP_FIN_WAIT1 (forget it, it's impossible) 2771 * TCP_CLOSE_WAIT -> TCP_LAST_ACK 2772 * 2773 * are legal only when FIN has been sent (i.e. in window), 2774 * rather than queued out of window. Purists blame. 2775 * 2776 * F.e. "RFC state" is ESTABLISHED, 2777 * if Linux state is FIN-WAIT-1, but FIN is still not sent. 2778 * 2779 * The visible declinations are that sometimes 2780 * we enter time-wait state, when it is not required really 2781 * (harmless), do not send active resets, when they are 2782 * required by specs (TCP_ESTABLISHED, TCP_CLOSE_WAIT, when 2783 * they look as CLOSING or LAST_ACK for Linux) 2784 * Probably, I missed some more holelets. 2785 * --ANK 2786 * XXX (TFO) - To start off we don't support SYN+ACK+FIN 2787 * in a single packet! (May consider it later but will 2788 * probably need API support or TCP_CORK SYN-ACK until 2789 * data is written and socket is closed.) 2790 */ 2791 tcp_send_fin(sk); 2792 } 2793 2794 sk_stream_wait_close(sk, timeout); 2795 2796 adjudge_to_death: 2797 state = sk->sk_state; 2798 sock_hold(sk); 2799 sock_orphan(sk); 2800 2801 local_bh_disable(); 2802 bh_lock_sock(sk); 2803 /* remove backlog if any, without releasing ownership. */ 2804 __release_sock(sk); 2805 2806 percpu_counter_inc(sk->sk_prot->orphan_count); 2807 2808 /* Have we already been destroyed by a softirq or backlog? */ 2809 if (state != TCP_CLOSE && sk->sk_state == TCP_CLOSE) 2810 goto out; 2811 2812 /* This is a (useful) BSD violating of the RFC. There is a 2813 * problem with TCP as specified in that the other end could 2814 * keep a socket open forever with no application left this end. 2815 * We use a 1 minute timeout (about the same as BSD) then kill 2816 * our end. If they send after that then tough - BUT: long enough 2817 * that we won't make the old 4*rto = almost no time - whoops 2818 * reset mistake. 2819 * 2820 * Nope, it was not mistake. It is really desired behaviour 2821 * f.e. on http servers, when such sockets are useless, but 2822 * consume significant resources. Let's do it with special 2823 * linger2 option. --ANK 2824 */ 2825 2826 if (sk->sk_state == TCP_FIN_WAIT2) { 2827 struct tcp_sock *tp = tcp_sk(sk); 2828 if (tp->linger2 < 0) { 2829 tcp_set_state(sk, TCP_CLOSE); 2830 tcp_send_active_reset(sk, GFP_ATOMIC); 2831 __NET_INC_STATS(sock_net(sk), 2832 LINUX_MIB_TCPABORTONLINGER); 2833 } else { 2834 const int tmo = tcp_fin_time(sk); 2835 2836 if (tmo > TCP_TIMEWAIT_LEN) { 2837 inet_csk_reset_keepalive_timer(sk, 2838 tmo - TCP_TIMEWAIT_LEN); 2839 } else { 2840 tcp_time_wait(sk, TCP_FIN_WAIT2, tmo); 2841 goto out; 2842 } 2843 } 2844 } 2845 if (sk->sk_state != TCP_CLOSE) { 2846 sk_mem_reclaim(sk); 2847 if (tcp_check_oom(sk, 0)) { 2848 tcp_set_state(sk, TCP_CLOSE); 2849 tcp_send_active_reset(sk, GFP_ATOMIC); 2850 __NET_INC_STATS(sock_net(sk), 2851 LINUX_MIB_TCPABORTONMEMORY); 2852 } else if (!check_net(sock_net(sk))) { 2853 /* Not possible to send reset; just close */ 2854 tcp_set_state(sk, TCP_CLOSE); 2855 } 2856 } 2857 2858 if (sk->sk_state == TCP_CLOSE) { 2859 struct request_sock *req; 2860 2861 req = rcu_dereference_protected(tcp_sk(sk)->fastopen_rsk, 2862 lockdep_sock_is_held(sk)); 2863 /* We could get here with a non-NULL req if the socket is 2864 * aborted (e.g., closed with unread data) before 3WHS 2865 * finishes. 2866 */ 2867 if (req) 2868 reqsk_fastopen_remove(sk, req, false); 2869 inet_csk_destroy_sock(sk); 2870 } 2871 /* Otherwise, socket is reprieved until protocol close. */ 2872 2873 out: 2874 bh_unlock_sock(sk); 2875 local_bh_enable(); 2876 } 2877 2878 void tcp_close(struct sock *sk, long timeout) 2879 { 2880 lock_sock(sk); 2881 __tcp_close(sk, timeout); 2882 release_sock(sk); 2883 sock_put(sk); 2884 } 2885 EXPORT_SYMBOL(tcp_close); 2886 2887 /* These states need RST on ABORT according to RFC793 */ 2888 2889 static inline bool tcp_need_reset(int state) 2890 { 2891 return (1 << state) & 2892 (TCPF_ESTABLISHED | TCPF_CLOSE_WAIT | TCPF_FIN_WAIT1 | 2893 TCPF_FIN_WAIT2 | TCPF_SYN_RECV); 2894 } 2895 2896 static void tcp_rtx_queue_purge(struct sock *sk) 2897 { 2898 struct rb_node *p = rb_first(&sk->tcp_rtx_queue); 2899 2900 tcp_sk(sk)->highest_sack = NULL; 2901 while (p) { 2902 struct sk_buff *skb = rb_to_skb(p); 2903 2904 p = rb_next(p); 2905 /* Since we are deleting whole queue, no need to 2906 * list_del(&skb->tcp_tsorted_anchor) 2907 */ 2908 tcp_rtx_queue_unlink(skb, sk); 2909 sk_wmem_free_skb(sk, skb); 2910 } 2911 } 2912 2913 void tcp_write_queue_purge(struct sock *sk) 2914 { 2915 struct sk_buff *skb; 2916 2917 tcp_chrono_stop(sk, TCP_CHRONO_BUSY); 2918 while ((skb = __skb_dequeue(&sk->sk_write_queue)) != NULL) { 2919 tcp_skb_tsorted_anchor_cleanup(skb); 2920 sk_wmem_free_skb(sk, skb); 2921 } 2922 tcp_rtx_queue_purge(sk); 2923 skb = sk->sk_tx_skb_cache; 2924 if (skb) { 2925 __kfree_skb(skb); 2926 sk->sk_tx_skb_cache = NULL; 2927 } 2928 INIT_LIST_HEAD(&tcp_sk(sk)->tsorted_sent_queue); 2929 sk_mem_reclaim(sk); 2930 tcp_clear_all_retrans_hints(tcp_sk(sk)); 2931 tcp_sk(sk)->packets_out = 0; 2932 inet_csk(sk)->icsk_backoff = 0; 2933 } 2934 2935 int tcp_disconnect(struct sock *sk, int flags) 2936 { 2937 struct inet_sock *inet = inet_sk(sk); 2938 struct inet_connection_sock *icsk = inet_csk(sk); 2939 struct tcp_sock *tp = tcp_sk(sk); 2940 int old_state = sk->sk_state; 2941 u32 seq; 2942 2943 if (old_state != TCP_CLOSE) 2944 tcp_set_state(sk, TCP_CLOSE); 2945 2946 /* ABORT function of RFC793 */ 2947 if (old_state == TCP_LISTEN) { 2948 inet_csk_listen_stop(sk); 2949 } else if (unlikely(tp->repair)) { 2950 sk->sk_err = ECONNABORTED; 2951 } else if (tcp_need_reset(old_state) || 2952 (tp->snd_nxt != tp->write_seq && 2953 (1 << old_state) & (TCPF_CLOSING | TCPF_LAST_ACK))) { 2954 /* The last check adjusts for discrepancy of Linux wrt. RFC 2955 * states 2956 */ 2957 tcp_send_active_reset(sk, gfp_any()); 2958 sk->sk_err = ECONNRESET; 2959 } else if (old_state == TCP_SYN_SENT) 2960 sk->sk_err = ECONNRESET; 2961 2962 tcp_clear_xmit_timers(sk); 2963 __skb_queue_purge(&sk->sk_receive_queue); 2964 if (sk->sk_rx_skb_cache) { 2965 __kfree_skb(sk->sk_rx_skb_cache); 2966 sk->sk_rx_skb_cache = NULL; 2967 } 2968 WRITE_ONCE(tp->copied_seq, tp->rcv_nxt); 2969 tp->urg_data = 0; 2970 tcp_write_queue_purge(sk); 2971 tcp_fastopen_active_disable_ofo_check(sk); 2972 skb_rbtree_purge(&tp->out_of_order_queue); 2973 2974 inet->inet_dport = 0; 2975 2976 if (!(sk->sk_userlocks & SOCK_BINDADDR_LOCK)) 2977 inet_reset_saddr(sk); 2978 2979 sk->sk_shutdown = 0; 2980 sock_reset_flag(sk, SOCK_DONE); 2981 tp->srtt_us = 0; 2982 tp->mdev_us = jiffies_to_usecs(TCP_TIMEOUT_INIT); 2983 tp->rcv_rtt_last_tsecr = 0; 2984 2985 seq = tp->write_seq + tp->max_window + 2; 2986 if (!seq) 2987 seq = 1; 2988 WRITE_ONCE(tp->write_seq, seq); 2989 2990 icsk->icsk_backoff = 0; 2991 icsk->icsk_probes_out = 0; 2992 icsk->icsk_probes_tstamp = 0; 2993 icsk->icsk_rto = TCP_TIMEOUT_INIT; 2994 icsk->icsk_rto_min = TCP_RTO_MIN; 2995 icsk->icsk_delack_max = TCP_DELACK_MAX; 2996 tp->snd_ssthresh = TCP_INFINITE_SSTHRESH; 2997 tp->snd_cwnd = TCP_INIT_CWND; 2998 tp->snd_cwnd_cnt = 0; 2999 tp->window_clamp = 0; 3000 tp->delivered = 0; 3001 tp->delivered_ce = 0; 3002 if (icsk->icsk_ca_ops->release) 3003 icsk->icsk_ca_ops->release(sk); 3004 memset(icsk->icsk_ca_priv, 0, sizeof(icsk->icsk_ca_priv)); 3005 icsk->icsk_ca_initialized = 0; 3006 tcp_set_ca_state(sk, TCP_CA_Open); 3007 tp->is_sack_reneg = 0; 3008 tcp_clear_retrans(tp); 3009 tp->total_retrans = 0; 3010 inet_csk_delack_init(sk); 3011 /* Initialize rcv_mss to TCP_MIN_MSS to avoid division by 0 3012 * issue in __tcp_select_window() 3013 */ 3014 icsk->icsk_ack.rcv_mss = TCP_MIN_MSS; 3015 memset(&tp->rx_opt, 0, sizeof(tp->rx_opt)); 3016 __sk_dst_reset(sk); 3017 dst_release(sk->sk_rx_dst); 3018 sk->sk_rx_dst = NULL; 3019 tcp_saved_syn_free(tp); 3020 tp->compressed_ack = 0; 3021 tp->segs_in = 0; 3022 tp->segs_out = 0; 3023 tp->bytes_sent = 0; 3024 tp->bytes_acked = 0; 3025 tp->bytes_received = 0; 3026 tp->bytes_retrans = 0; 3027 tp->data_segs_in = 0; 3028 tp->data_segs_out = 0; 3029 tp->duplicate_sack[0].start_seq = 0; 3030 tp->duplicate_sack[0].end_seq = 0; 3031 tp->dsack_dups = 0; 3032 tp->reord_seen = 0; 3033 tp->retrans_out = 0; 3034 tp->sacked_out = 0; 3035 tp->tlp_high_seq = 0; 3036 tp->last_oow_ack_time = 0; 3037 /* There's a bubble in the pipe until at least the first ACK. */ 3038 tp->app_limited = ~0U; 3039 tp->rack.mstamp = 0; 3040 tp->rack.advanced = 0; 3041 tp->rack.reo_wnd_steps = 1; 3042 tp->rack.last_delivered = 0; 3043 tp->rack.reo_wnd_persist = 0; 3044 tp->rack.dsack_seen = 0; 3045 tp->syn_data_acked = 0; 3046 tp->rx_opt.saw_tstamp = 0; 3047 tp->rx_opt.dsack = 0; 3048 tp->rx_opt.num_sacks = 0; 3049 tp->rcv_ooopack = 0; 3050 3051 3052 /* Clean up fastopen related fields */ 3053 tcp_free_fastopen_req(tp); 3054 inet->defer_connect = 0; 3055 tp->fastopen_client_fail = 0; 3056 3057 WARN_ON(inet->inet_num && !icsk->icsk_bind_hash); 3058 3059 if (sk->sk_frag.page) { 3060 put_page(sk->sk_frag.page); 3061 sk->sk_frag.page = NULL; 3062 sk->sk_frag.offset = 0; 3063 } 3064 3065 sk_error_report(sk); 3066 return 0; 3067 } 3068 EXPORT_SYMBOL(tcp_disconnect); 3069 3070 static inline bool tcp_can_repair_sock(const struct sock *sk) 3071 { 3072 return ns_capable(sock_net(sk)->user_ns, CAP_NET_ADMIN) && 3073 (sk->sk_state != TCP_LISTEN); 3074 } 3075 3076 static int tcp_repair_set_window(struct tcp_sock *tp, sockptr_t optbuf, int len) 3077 { 3078 struct tcp_repair_window opt; 3079 3080 if (!tp->repair) 3081 return -EPERM; 3082 3083 if (len != sizeof(opt)) 3084 return -EINVAL; 3085 3086 if (copy_from_sockptr(&opt, optbuf, sizeof(opt))) 3087 return -EFAULT; 3088 3089 if (opt.max_window < opt.snd_wnd) 3090 return -EINVAL; 3091 3092 if (after(opt.snd_wl1, tp->rcv_nxt + opt.rcv_wnd)) 3093 return -EINVAL; 3094 3095 if (after(opt.rcv_wup, tp->rcv_nxt)) 3096 return -EINVAL; 3097 3098 tp->snd_wl1 = opt.snd_wl1; 3099 tp->snd_wnd = opt.snd_wnd; 3100 tp->max_window = opt.max_window; 3101 3102 tp->rcv_wnd = opt.rcv_wnd; 3103 tp->rcv_wup = opt.rcv_wup; 3104 3105 return 0; 3106 } 3107 3108 static int tcp_repair_options_est(struct sock *sk, sockptr_t optbuf, 3109 unsigned int len) 3110 { 3111 struct tcp_sock *tp = tcp_sk(sk); 3112 struct tcp_repair_opt opt; 3113 size_t offset = 0; 3114 3115 while (len >= sizeof(opt)) { 3116 if (copy_from_sockptr_offset(&opt, optbuf, offset, sizeof(opt))) 3117 return -EFAULT; 3118 3119 offset += sizeof(opt); 3120 len -= sizeof(opt); 3121 3122 switch (opt.opt_code) { 3123 case TCPOPT_MSS: 3124 tp->rx_opt.mss_clamp = opt.opt_val; 3125 tcp_mtup_init(sk); 3126 break; 3127 case TCPOPT_WINDOW: 3128 { 3129 u16 snd_wscale = opt.opt_val & 0xFFFF; 3130 u16 rcv_wscale = opt.opt_val >> 16; 3131 3132 if (snd_wscale > TCP_MAX_WSCALE || rcv_wscale > TCP_MAX_WSCALE) 3133 return -EFBIG; 3134 3135 tp->rx_opt.snd_wscale = snd_wscale; 3136 tp->rx_opt.rcv_wscale = rcv_wscale; 3137 tp->rx_opt.wscale_ok = 1; 3138 } 3139 break; 3140 case TCPOPT_SACK_PERM: 3141 if (opt.opt_val != 0) 3142 return -EINVAL; 3143 3144 tp->rx_opt.sack_ok |= TCP_SACK_SEEN; 3145 break; 3146 case TCPOPT_TIMESTAMP: 3147 if (opt.opt_val != 0) 3148 return -EINVAL; 3149 3150 tp->rx_opt.tstamp_ok = 1; 3151 break; 3152 } 3153 } 3154 3155 return 0; 3156 } 3157 3158 DEFINE_STATIC_KEY_FALSE(tcp_tx_delay_enabled); 3159 EXPORT_SYMBOL(tcp_tx_delay_enabled); 3160 3161 static void tcp_enable_tx_delay(void) 3162 { 3163 if (!static_branch_unlikely(&tcp_tx_delay_enabled)) { 3164 static int __tcp_tx_delay_enabled = 0; 3165 3166 if (cmpxchg(&__tcp_tx_delay_enabled, 0, 1) == 0) { 3167 static_branch_enable(&tcp_tx_delay_enabled); 3168 pr_info("TCP_TX_DELAY enabled\n"); 3169 } 3170 } 3171 } 3172 3173 /* When set indicates to always queue non-full frames. Later the user clears 3174 * this option and we transmit any pending partial frames in the queue. This is 3175 * meant to be used alongside sendfile() to get properly filled frames when the 3176 * user (for example) must write out headers with a write() call first and then 3177 * use sendfile to send out the data parts. 3178 * 3179 * TCP_CORK can be set together with TCP_NODELAY and it is stronger than 3180 * TCP_NODELAY. 3181 */ 3182 static void __tcp_sock_set_cork(struct sock *sk, bool on) 3183 { 3184 struct tcp_sock *tp = tcp_sk(sk); 3185 3186 if (on) { 3187 tp->nonagle |= TCP_NAGLE_CORK; 3188 } else { 3189 tp->nonagle &= ~TCP_NAGLE_CORK; 3190 if (tp->nonagle & TCP_NAGLE_OFF) 3191 tp->nonagle |= TCP_NAGLE_PUSH; 3192 tcp_push_pending_frames(sk); 3193 } 3194 } 3195 3196 void tcp_sock_set_cork(struct sock *sk, bool on) 3197 { 3198 lock_sock(sk); 3199 __tcp_sock_set_cork(sk, on); 3200 release_sock(sk); 3201 } 3202 EXPORT_SYMBOL(tcp_sock_set_cork); 3203 3204 /* TCP_NODELAY is weaker than TCP_CORK, so that this option on corked socket is 3205 * remembered, but it is not activated until cork is cleared. 3206 * 3207 * However, when TCP_NODELAY is set we make an explicit push, which overrides 3208 * even TCP_CORK for currently queued segments. 3209 */ 3210 static void __tcp_sock_set_nodelay(struct sock *sk, bool on) 3211 { 3212 if (on) { 3213 tcp_sk(sk)->nonagle |= TCP_NAGLE_OFF|TCP_NAGLE_PUSH; 3214 tcp_push_pending_frames(sk); 3215 } else { 3216 tcp_sk(sk)->nonagle &= ~TCP_NAGLE_OFF; 3217 } 3218 } 3219 3220 void tcp_sock_set_nodelay(struct sock *sk) 3221 { 3222 lock_sock(sk); 3223 __tcp_sock_set_nodelay(sk, true); 3224 release_sock(sk); 3225 } 3226 EXPORT_SYMBOL(tcp_sock_set_nodelay); 3227 3228 static void __tcp_sock_set_quickack(struct sock *sk, int val) 3229 { 3230 if (!val) { 3231 inet_csk_enter_pingpong_mode(sk); 3232 return; 3233 } 3234 3235 inet_csk_exit_pingpong_mode(sk); 3236 if ((1 << sk->sk_state) & (TCPF_ESTABLISHED | TCPF_CLOSE_WAIT) && 3237 inet_csk_ack_scheduled(sk)) { 3238 inet_csk(sk)->icsk_ack.pending |= ICSK_ACK_PUSHED; 3239 tcp_cleanup_rbuf(sk, 1); 3240 if (!(val & 1)) 3241 inet_csk_enter_pingpong_mode(sk); 3242 } 3243 } 3244 3245 void tcp_sock_set_quickack(struct sock *sk, int val) 3246 { 3247 lock_sock(sk); 3248 __tcp_sock_set_quickack(sk, val); 3249 release_sock(sk); 3250 } 3251 EXPORT_SYMBOL(tcp_sock_set_quickack); 3252 3253 int tcp_sock_set_syncnt(struct sock *sk, int val) 3254 { 3255 if (val < 1 || val > MAX_TCP_SYNCNT) 3256 return -EINVAL; 3257 3258 lock_sock(sk); 3259 inet_csk(sk)->icsk_syn_retries = val; 3260 release_sock(sk); 3261 return 0; 3262 } 3263 EXPORT_SYMBOL(tcp_sock_set_syncnt); 3264 3265 void tcp_sock_set_user_timeout(struct sock *sk, u32 val) 3266 { 3267 lock_sock(sk); 3268 inet_csk(sk)->icsk_user_timeout = val; 3269 release_sock(sk); 3270 } 3271 EXPORT_SYMBOL(tcp_sock_set_user_timeout); 3272 3273 int tcp_sock_set_keepidle_locked(struct sock *sk, int val) 3274 { 3275 struct tcp_sock *tp = tcp_sk(sk); 3276 3277 if (val < 1 || val > MAX_TCP_KEEPIDLE) 3278 return -EINVAL; 3279 3280 tp->keepalive_time = val * HZ; 3281 if (sock_flag(sk, SOCK_KEEPOPEN) && 3282 !((1 << sk->sk_state) & (TCPF_CLOSE | TCPF_LISTEN))) { 3283 u32 elapsed = keepalive_time_elapsed(tp); 3284 3285 if (tp->keepalive_time > elapsed) 3286 elapsed = tp->keepalive_time - elapsed; 3287 else 3288 elapsed = 0; 3289 inet_csk_reset_keepalive_timer(sk, elapsed); 3290 } 3291 3292 return 0; 3293 } 3294 3295 int tcp_sock_set_keepidle(struct sock *sk, int val) 3296 { 3297 int err; 3298 3299 lock_sock(sk); 3300 err = tcp_sock_set_keepidle_locked(sk, val); 3301 release_sock(sk); 3302 return err; 3303 } 3304 EXPORT_SYMBOL(tcp_sock_set_keepidle); 3305 3306 int tcp_sock_set_keepintvl(struct sock *sk, int val) 3307 { 3308 if (val < 1 || val > MAX_TCP_KEEPINTVL) 3309 return -EINVAL; 3310 3311 lock_sock(sk); 3312 tcp_sk(sk)->keepalive_intvl = val * HZ; 3313 release_sock(sk); 3314 return 0; 3315 } 3316 EXPORT_SYMBOL(tcp_sock_set_keepintvl); 3317 3318 int tcp_sock_set_keepcnt(struct sock *sk, int val) 3319 { 3320 if (val < 1 || val > MAX_TCP_KEEPCNT) 3321 return -EINVAL; 3322 3323 lock_sock(sk); 3324 tcp_sk(sk)->keepalive_probes = val; 3325 release_sock(sk); 3326 return 0; 3327 } 3328 EXPORT_SYMBOL(tcp_sock_set_keepcnt); 3329 3330 int tcp_set_window_clamp(struct sock *sk, int val) 3331 { 3332 struct tcp_sock *tp = tcp_sk(sk); 3333 3334 if (!val) { 3335 if (sk->sk_state != TCP_CLOSE) 3336 return -EINVAL; 3337 tp->window_clamp = 0; 3338 } else { 3339 tp->window_clamp = val < SOCK_MIN_RCVBUF / 2 ? 3340 SOCK_MIN_RCVBUF / 2 : val; 3341 } 3342 return 0; 3343 } 3344 3345 /* 3346 * Socket option code for TCP. 3347 */ 3348 static int do_tcp_setsockopt(struct sock *sk, int level, int optname, 3349 sockptr_t optval, unsigned int optlen) 3350 { 3351 struct tcp_sock *tp = tcp_sk(sk); 3352 struct inet_connection_sock *icsk = inet_csk(sk); 3353 struct net *net = sock_net(sk); 3354 int val; 3355 int err = 0; 3356 3357 /* These are data/string values, all the others are ints */ 3358 switch (optname) { 3359 case TCP_CONGESTION: { 3360 char name[TCP_CA_NAME_MAX]; 3361 3362 if (optlen < 1) 3363 return -EINVAL; 3364 3365 val = strncpy_from_sockptr(name, optval, 3366 min_t(long, TCP_CA_NAME_MAX-1, optlen)); 3367 if (val < 0) 3368 return -EFAULT; 3369 name[val] = 0; 3370 3371 lock_sock(sk); 3372 err = tcp_set_congestion_control(sk, name, true, 3373 ns_capable(sock_net(sk)->user_ns, 3374 CAP_NET_ADMIN)); 3375 release_sock(sk); 3376 return err; 3377 } 3378 case TCP_ULP: { 3379 char name[TCP_ULP_NAME_MAX]; 3380 3381 if (optlen < 1) 3382 return -EINVAL; 3383 3384 val = strncpy_from_sockptr(name, optval, 3385 min_t(long, TCP_ULP_NAME_MAX - 1, 3386 optlen)); 3387 if (val < 0) 3388 return -EFAULT; 3389 name[val] = 0; 3390 3391 lock_sock(sk); 3392 err = tcp_set_ulp(sk, name); 3393 release_sock(sk); 3394 return err; 3395 } 3396 case TCP_FASTOPEN_KEY: { 3397 __u8 key[TCP_FASTOPEN_KEY_BUF_LENGTH]; 3398 __u8 *backup_key = NULL; 3399 3400 /* Allow a backup key as well to facilitate key rotation 3401 * First key is the active one. 3402 */ 3403 if (optlen != TCP_FASTOPEN_KEY_LENGTH && 3404 optlen != TCP_FASTOPEN_KEY_BUF_LENGTH) 3405 return -EINVAL; 3406 3407 if (copy_from_sockptr(key, optval, optlen)) 3408 return -EFAULT; 3409 3410 if (optlen == TCP_FASTOPEN_KEY_BUF_LENGTH) 3411 backup_key = key + TCP_FASTOPEN_KEY_LENGTH; 3412 3413 return tcp_fastopen_reset_cipher(net, sk, key, backup_key); 3414 } 3415 default: 3416 /* fallthru */ 3417 break; 3418 } 3419 3420 if (optlen < sizeof(int)) 3421 return -EINVAL; 3422 3423 if (copy_from_sockptr(&val, optval, sizeof(val))) 3424 return -EFAULT; 3425 3426 lock_sock(sk); 3427 3428 switch (optname) { 3429 case TCP_MAXSEG: 3430 /* Values greater than interface MTU won't take effect. However 3431 * at the point when this call is done we typically don't yet 3432 * know which interface is going to be used 3433 */ 3434 if (val && (val < TCP_MIN_MSS || val > MAX_TCP_WINDOW)) { 3435 err = -EINVAL; 3436 break; 3437 } 3438 tp->rx_opt.user_mss = val; 3439 break; 3440 3441 case TCP_NODELAY: 3442 __tcp_sock_set_nodelay(sk, val); 3443 break; 3444 3445 case TCP_THIN_LINEAR_TIMEOUTS: 3446 if (val < 0 || val > 1) 3447 err = -EINVAL; 3448 else 3449 tp->thin_lto = val; 3450 break; 3451 3452 case TCP_THIN_DUPACK: 3453 if (val < 0 || val > 1) 3454 err = -EINVAL; 3455 break; 3456 3457 case TCP_REPAIR: 3458 if (!tcp_can_repair_sock(sk)) 3459 err = -EPERM; 3460 else if (val == TCP_REPAIR_ON) { 3461 tp->repair = 1; 3462 sk->sk_reuse = SK_FORCE_REUSE; 3463 tp->repair_queue = TCP_NO_QUEUE; 3464 } else if (val == TCP_REPAIR_OFF) { 3465 tp->repair = 0; 3466 sk->sk_reuse = SK_NO_REUSE; 3467 tcp_send_window_probe(sk); 3468 } else if (val == TCP_REPAIR_OFF_NO_WP) { 3469 tp->repair = 0; 3470 sk->sk_reuse = SK_NO_REUSE; 3471 } else 3472 err = -EINVAL; 3473 3474 break; 3475 3476 case TCP_REPAIR_QUEUE: 3477 if (!tp->repair) 3478 err = -EPERM; 3479 else if ((unsigned int)val < TCP_QUEUES_NR) 3480 tp->repair_queue = val; 3481 else 3482 err = -EINVAL; 3483 break; 3484 3485 case TCP_QUEUE_SEQ: 3486 if (sk->sk_state != TCP_CLOSE) { 3487 err = -EPERM; 3488 } else if (tp->repair_queue == TCP_SEND_QUEUE) { 3489 if (!tcp_rtx_queue_empty(sk)) 3490 err = -EPERM; 3491 else 3492 WRITE_ONCE(tp->write_seq, val); 3493 } else if (tp->repair_queue == TCP_RECV_QUEUE) { 3494 if (tp->rcv_nxt != tp->copied_seq) { 3495 err = -EPERM; 3496 } else { 3497 WRITE_ONCE(tp->rcv_nxt, val); 3498 WRITE_ONCE(tp->copied_seq, val); 3499 } 3500 } else { 3501 err = -EINVAL; 3502 } 3503 break; 3504 3505 case TCP_REPAIR_OPTIONS: 3506 if (!tp->repair) 3507 err = -EINVAL; 3508 else if (sk->sk_state == TCP_ESTABLISHED) 3509 err = tcp_repair_options_est(sk, optval, optlen); 3510 else 3511 err = -EPERM; 3512 break; 3513 3514 case TCP_CORK: 3515 __tcp_sock_set_cork(sk, val); 3516 break; 3517 3518 case TCP_KEEPIDLE: 3519 err = tcp_sock_set_keepidle_locked(sk, val); 3520 break; 3521 case TCP_KEEPINTVL: 3522 if (val < 1 || val > MAX_TCP_KEEPINTVL) 3523 err = -EINVAL; 3524 else 3525 tp->keepalive_intvl = val * HZ; 3526 break; 3527 case TCP_KEEPCNT: 3528 if (val < 1 || val > MAX_TCP_KEEPCNT) 3529 err = -EINVAL; 3530 else 3531 tp->keepalive_probes = val; 3532 break; 3533 case TCP_SYNCNT: 3534 if (val < 1 || val > MAX_TCP_SYNCNT) 3535 err = -EINVAL; 3536 else 3537 icsk->icsk_syn_retries = val; 3538 break; 3539 3540 case TCP_SAVE_SYN: 3541 /* 0: disable, 1: enable, 2: start from ether_header */ 3542 if (val < 0 || val > 2) 3543 err = -EINVAL; 3544 else 3545 tp->save_syn = val; 3546 break; 3547 3548 case TCP_LINGER2: 3549 if (val < 0) 3550 tp->linger2 = -1; 3551 else if (val > TCP_FIN_TIMEOUT_MAX / HZ) 3552 tp->linger2 = TCP_FIN_TIMEOUT_MAX; 3553 else 3554 tp->linger2 = val * HZ; 3555 break; 3556 3557 case TCP_DEFER_ACCEPT: 3558 /* Translate value in seconds to number of retransmits */ 3559 icsk->icsk_accept_queue.rskq_defer_accept = 3560 secs_to_retrans(val, TCP_TIMEOUT_INIT / HZ, 3561 TCP_RTO_MAX / HZ); 3562 break; 3563 3564 case TCP_WINDOW_CLAMP: 3565 err = tcp_set_window_clamp(sk, val); 3566 break; 3567 3568 case TCP_QUICKACK: 3569 __tcp_sock_set_quickack(sk, val); 3570 break; 3571 3572 #ifdef CONFIG_TCP_MD5SIG 3573 case TCP_MD5SIG: 3574 case TCP_MD5SIG_EXT: 3575 err = tp->af_specific->md5_parse(sk, optname, optval, optlen); 3576 break; 3577 #endif 3578 case TCP_USER_TIMEOUT: 3579 /* Cap the max time in ms TCP will retry or probe the window 3580 * before giving up and aborting (ETIMEDOUT) a connection. 3581 */ 3582 if (val < 0) 3583 err = -EINVAL; 3584 else 3585 icsk->icsk_user_timeout = val; 3586 break; 3587 3588 case TCP_FASTOPEN: 3589 if (val >= 0 && ((1 << sk->sk_state) & (TCPF_CLOSE | 3590 TCPF_LISTEN))) { 3591 tcp_fastopen_init_key_once(net); 3592 3593 fastopen_queue_tune(sk, val); 3594 } else { 3595 err = -EINVAL; 3596 } 3597 break; 3598 case TCP_FASTOPEN_CONNECT: 3599 if (val > 1 || val < 0) { 3600 err = -EINVAL; 3601 } else if (net->ipv4.sysctl_tcp_fastopen & TFO_CLIENT_ENABLE) { 3602 if (sk->sk_state == TCP_CLOSE) 3603 tp->fastopen_connect = val; 3604 else 3605 err = -EINVAL; 3606 } else { 3607 err = -EOPNOTSUPP; 3608 } 3609 break; 3610 case TCP_FASTOPEN_NO_COOKIE: 3611 if (val > 1 || val < 0) 3612 err = -EINVAL; 3613 else if (!((1 << sk->sk_state) & (TCPF_CLOSE | TCPF_LISTEN))) 3614 err = -EINVAL; 3615 else 3616 tp->fastopen_no_cookie = val; 3617 break; 3618 case TCP_TIMESTAMP: 3619 if (!tp->repair) 3620 err = -EPERM; 3621 else 3622 tp->tsoffset = val - tcp_time_stamp_raw(); 3623 break; 3624 case TCP_REPAIR_WINDOW: 3625 err = tcp_repair_set_window(tp, optval, optlen); 3626 break; 3627 case TCP_NOTSENT_LOWAT: 3628 tp->notsent_lowat = val; 3629 sk->sk_write_space(sk); 3630 break; 3631 case TCP_INQ: 3632 if (val > 1 || val < 0) 3633 err = -EINVAL; 3634 else 3635 tp->recvmsg_inq = val; 3636 break; 3637 case TCP_TX_DELAY: 3638 if (val) 3639 tcp_enable_tx_delay(); 3640 tp->tcp_tx_delay = val; 3641 break; 3642 default: 3643 err = -ENOPROTOOPT; 3644 break; 3645 } 3646 3647 release_sock(sk); 3648 return err; 3649 } 3650 3651 int tcp_setsockopt(struct sock *sk, int level, int optname, sockptr_t optval, 3652 unsigned int optlen) 3653 { 3654 const struct inet_connection_sock *icsk = inet_csk(sk); 3655 3656 if (level != SOL_TCP) 3657 return icsk->icsk_af_ops->setsockopt(sk, level, optname, 3658 optval, optlen); 3659 return do_tcp_setsockopt(sk, level, optname, optval, optlen); 3660 } 3661 EXPORT_SYMBOL(tcp_setsockopt); 3662 3663 static void tcp_get_info_chrono_stats(const struct tcp_sock *tp, 3664 struct tcp_info *info) 3665 { 3666 u64 stats[__TCP_CHRONO_MAX], total = 0; 3667 enum tcp_chrono i; 3668 3669 for (i = TCP_CHRONO_BUSY; i < __TCP_CHRONO_MAX; ++i) { 3670 stats[i] = tp->chrono_stat[i - 1]; 3671 if (i == tp->chrono_type) 3672 stats[i] += tcp_jiffies32 - tp->chrono_start; 3673 stats[i] *= USEC_PER_SEC / HZ; 3674 total += stats[i]; 3675 } 3676 3677 info->tcpi_busy_time = total; 3678 info->tcpi_rwnd_limited = stats[TCP_CHRONO_RWND_LIMITED]; 3679 info->tcpi_sndbuf_limited = stats[TCP_CHRONO_SNDBUF_LIMITED]; 3680 } 3681 3682 /* Return information about state of tcp endpoint in API format. */ 3683 void tcp_get_info(struct sock *sk, struct tcp_info *info) 3684 { 3685 const struct tcp_sock *tp = tcp_sk(sk); /* iff sk_type == SOCK_STREAM */ 3686 const struct inet_connection_sock *icsk = inet_csk(sk); 3687 unsigned long rate; 3688 u32 now; 3689 u64 rate64; 3690 bool slow; 3691 3692 memset(info, 0, sizeof(*info)); 3693 if (sk->sk_type != SOCK_STREAM) 3694 return; 3695 3696 info->tcpi_state = inet_sk_state_load(sk); 3697 3698 /* Report meaningful fields for all TCP states, including listeners */ 3699 rate = READ_ONCE(sk->sk_pacing_rate); 3700 rate64 = (rate != ~0UL) ? rate : ~0ULL; 3701 info->tcpi_pacing_rate = rate64; 3702 3703 rate = READ_ONCE(sk->sk_max_pacing_rate); 3704 rate64 = (rate != ~0UL) ? rate : ~0ULL; 3705 info->tcpi_max_pacing_rate = rate64; 3706 3707 info->tcpi_reordering = tp->reordering; 3708 info->tcpi_snd_cwnd = tp->snd_cwnd; 3709 3710 if (info->tcpi_state == TCP_LISTEN) { 3711 /* listeners aliased fields : 3712 * tcpi_unacked -> Number of children ready for accept() 3713 * tcpi_sacked -> max backlog 3714 */ 3715 info->tcpi_unacked = READ_ONCE(sk->sk_ack_backlog); 3716 info->tcpi_sacked = READ_ONCE(sk->sk_max_ack_backlog); 3717 return; 3718 } 3719 3720 slow = lock_sock_fast(sk); 3721 3722 info->tcpi_ca_state = icsk->icsk_ca_state; 3723 info->tcpi_retransmits = icsk->icsk_retransmits; 3724 info->tcpi_probes = icsk->icsk_probes_out; 3725 info->tcpi_backoff = icsk->icsk_backoff; 3726 3727 if (tp->rx_opt.tstamp_ok) 3728 info->tcpi_options |= TCPI_OPT_TIMESTAMPS; 3729 if (tcp_is_sack(tp)) 3730 info->tcpi_options |= TCPI_OPT_SACK; 3731 if (tp->rx_opt.wscale_ok) { 3732 info->tcpi_options |= TCPI_OPT_WSCALE; 3733 info->tcpi_snd_wscale = tp->rx_opt.snd_wscale; 3734 info->tcpi_rcv_wscale = tp->rx_opt.rcv_wscale; 3735 } 3736 3737 if (tp->ecn_flags & TCP_ECN_OK) 3738 info->tcpi_options |= TCPI_OPT_ECN; 3739 if (tp->ecn_flags & TCP_ECN_SEEN) 3740 info->tcpi_options |= TCPI_OPT_ECN_SEEN; 3741 if (tp->syn_data_acked) 3742 info->tcpi_options |= TCPI_OPT_SYN_DATA; 3743 3744 info->tcpi_rto = jiffies_to_usecs(icsk->icsk_rto); 3745 info->tcpi_ato = jiffies_to_usecs(icsk->icsk_ack.ato); 3746 info->tcpi_snd_mss = tp->mss_cache; 3747 info->tcpi_rcv_mss = icsk->icsk_ack.rcv_mss; 3748 3749 info->tcpi_unacked = tp->packets_out; 3750 info->tcpi_sacked = tp->sacked_out; 3751 3752 info->tcpi_lost = tp->lost_out; 3753 info->tcpi_retrans = tp->retrans_out; 3754 3755 now = tcp_jiffies32; 3756 info->tcpi_last_data_sent = jiffies_to_msecs(now - tp->lsndtime); 3757 info->tcpi_last_data_recv = jiffies_to_msecs(now - icsk->icsk_ack.lrcvtime); 3758 info->tcpi_last_ack_recv = jiffies_to_msecs(now - tp->rcv_tstamp); 3759 3760 info->tcpi_pmtu = icsk->icsk_pmtu_cookie; 3761 info->tcpi_rcv_ssthresh = tp->rcv_ssthresh; 3762 info->tcpi_rtt = tp->srtt_us >> 3; 3763 info->tcpi_rttvar = tp->mdev_us >> 2; 3764 info->tcpi_snd_ssthresh = tp->snd_ssthresh; 3765 info->tcpi_advmss = tp->advmss; 3766 3767 info->tcpi_rcv_rtt = tp->rcv_rtt_est.rtt_us >> 3; 3768 info->tcpi_rcv_space = tp->rcvq_space.space; 3769 3770 info->tcpi_total_retrans = tp->total_retrans; 3771 3772 info->tcpi_bytes_acked = tp->bytes_acked; 3773 info->tcpi_bytes_received = tp->bytes_received; 3774 info->tcpi_notsent_bytes = max_t(int, 0, tp->write_seq - tp->snd_nxt); 3775 tcp_get_info_chrono_stats(tp, info); 3776 3777 info->tcpi_segs_out = tp->segs_out; 3778 info->tcpi_segs_in = tp->segs_in; 3779 3780 info->tcpi_min_rtt = tcp_min_rtt(tp); 3781 info->tcpi_data_segs_in = tp->data_segs_in; 3782 info->tcpi_data_segs_out = tp->data_segs_out; 3783 3784 info->tcpi_delivery_rate_app_limited = tp->rate_app_limited ? 1 : 0; 3785 rate64 = tcp_compute_delivery_rate(tp); 3786 if (rate64) 3787 info->tcpi_delivery_rate = rate64; 3788 info->tcpi_delivered = tp->delivered; 3789 info->tcpi_delivered_ce = tp->delivered_ce; 3790 info->tcpi_bytes_sent = tp->bytes_sent; 3791 info->tcpi_bytes_retrans = tp->bytes_retrans; 3792 info->tcpi_dsack_dups = tp->dsack_dups; 3793 info->tcpi_reord_seen = tp->reord_seen; 3794 info->tcpi_rcv_ooopack = tp->rcv_ooopack; 3795 info->tcpi_snd_wnd = tp->snd_wnd; 3796 info->tcpi_fastopen_client_fail = tp->fastopen_client_fail; 3797 unlock_sock_fast(sk, slow); 3798 } 3799 EXPORT_SYMBOL_GPL(tcp_get_info); 3800 3801 static size_t tcp_opt_stats_get_size(void) 3802 { 3803 return 3804 nla_total_size_64bit(sizeof(u64)) + /* TCP_NLA_BUSY */ 3805 nla_total_size_64bit(sizeof(u64)) + /* TCP_NLA_RWND_LIMITED */ 3806 nla_total_size_64bit(sizeof(u64)) + /* TCP_NLA_SNDBUF_LIMITED */ 3807 nla_total_size_64bit(sizeof(u64)) + /* TCP_NLA_DATA_SEGS_OUT */ 3808 nla_total_size_64bit(sizeof(u64)) + /* TCP_NLA_TOTAL_RETRANS */ 3809 nla_total_size_64bit(sizeof(u64)) + /* TCP_NLA_PACING_RATE */ 3810 nla_total_size_64bit(sizeof(u64)) + /* TCP_NLA_DELIVERY_RATE */ 3811 nla_total_size(sizeof(u32)) + /* TCP_NLA_SND_CWND */ 3812 nla_total_size(sizeof(u32)) + /* TCP_NLA_REORDERING */ 3813 nla_total_size(sizeof(u32)) + /* TCP_NLA_MIN_RTT */ 3814 nla_total_size(sizeof(u8)) + /* TCP_NLA_RECUR_RETRANS */ 3815 nla_total_size(sizeof(u8)) + /* TCP_NLA_DELIVERY_RATE_APP_LMT */ 3816 nla_total_size(sizeof(u32)) + /* TCP_NLA_SNDQ_SIZE */ 3817 nla_total_size(sizeof(u8)) + /* TCP_NLA_CA_STATE */ 3818 nla_total_size(sizeof(u32)) + /* TCP_NLA_SND_SSTHRESH */ 3819 nla_total_size(sizeof(u32)) + /* TCP_NLA_DELIVERED */ 3820 nla_total_size(sizeof(u32)) + /* TCP_NLA_DELIVERED_CE */ 3821 nla_total_size_64bit(sizeof(u64)) + /* TCP_NLA_BYTES_SENT */ 3822 nla_total_size_64bit(sizeof(u64)) + /* TCP_NLA_BYTES_RETRANS */ 3823 nla_total_size(sizeof(u32)) + /* TCP_NLA_DSACK_DUPS */ 3824 nla_total_size(sizeof(u32)) + /* TCP_NLA_REORD_SEEN */ 3825 nla_total_size(sizeof(u32)) + /* TCP_NLA_SRTT */ 3826 nla_total_size(sizeof(u16)) + /* TCP_NLA_TIMEOUT_REHASH */ 3827 nla_total_size(sizeof(u32)) + /* TCP_NLA_BYTES_NOTSENT */ 3828 nla_total_size_64bit(sizeof(u64)) + /* TCP_NLA_EDT */ 3829 nla_total_size(sizeof(u8)) + /* TCP_NLA_TTL */ 3830 0; 3831 } 3832 3833 /* Returns TTL or hop limit of an incoming packet from skb. */ 3834 static u8 tcp_skb_ttl_or_hop_limit(const struct sk_buff *skb) 3835 { 3836 if (skb->protocol == htons(ETH_P_IP)) 3837 return ip_hdr(skb)->ttl; 3838 else if (skb->protocol == htons(ETH_P_IPV6)) 3839 return ipv6_hdr(skb)->hop_limit; 3840 else 3841 return 0; 3842 } 3843 3844 struct sk_buff *tcp_get_timestamping_opt_stats(const struct sock *sk, 3845 const struct sk_buff *orig_skb, 3846 const struct sk_buff *ack_skb) 3847 { 3848 const struct tcp_sock *tp = tcp_sk(sk); 3849 struct sk_buff *stats; 3850 struct tcp_info info; 3851 unsigned long rate; 3852 u64 rate64; 3853 3854 stats = alloc_skb(tcp_opt_stats_get_size(), GFP_ATOMIC); 3855 if (!stats) 3856 return NULL; 3857 3858 tcp_get_info_chrono_stats(tp, &info); 3859 nla_put_u64_64bit(stats, TCP_NLA_BUSY, 3860 info.tcpi_busy_time, TCP_NLA_PAD); 3861 nla_put_u64_64bit(stats, TCP_NLA_RWND_LIMITED, 3862 info.tcpi_rwnd_limited, TCP_NLA_PAD); 3863 nla_put_u64_64bit(stats, TCP_NLA_SNDBUF_LIMITED, 3864 info.tcpi_sndbuf_limited, TCP_NLA_PAD); 3865 nla_put_u64_64bit(stats, TCP_NLA_DATA_SEGS_OUT, 3866 tp->data_segs_out, TCP_NLA_PAD); 3867 nla_put_u64_64bit(stats, TCP_NLA_TOTAL_RETRANS, 3868 tp->total_retrans, TCP_NLA_PAD); 3869 3870 rate = READ_ONCE(sk->sk_pacing_rate); 3871 rate64 = (rate != ~0UL) ? rate : ~0ULL; 3872 nla_put_u64_64bit(stats, TCP_NLA_PACING_RATE, rate64, TCP_NLA_PAD); 3873 3874 rate64 = tcp_compute_delivery_rate(tp); 3875 nla_put_u64_64bit(stats, TCP_NLA_DELIVERY_RATE, rate64, TCP_NLA_PAD); 3876 3877 nla_put_u32(stats, TCP_NLA_SND_CWND, tp->snd_cwnd); 3878 nla_put_u32(stats, TCP_NLA_REORDERING, tp->reordering); 3879 nla_put_u32(stats, TCP_NLA_MIN_RTT, tcp_min_rtt(tp)); 3880 3881 nla_put_u8(stats, TCP_NLA_RECUR_RETRANS, inet_csk(sk)->icsk_retransmits); 3882 nla_put_u8(stats, TCP_NLA_DELIVERY_RATE_APP_LMT, !!tp->rate_app_limited); 3883 nla_put_u32(stats, TCP_NLA_SND_SSTHRESH, tp->snd_ssthresh); 3884 nla_put_u32(stats, TCP_NLA_DELIVERED, tp->delivered); 3885 nla_put_u32(stats, TCP_NLA_DELIVERED_CE, tp->delivered_ce); 3886 3887 nla_put_u32(stats, TCP_NLA_SNDQ_SIZE, tp->write_seq - tp->snd_una); 3888 nla_put_u8(stats, TCP_NLA_CA_STATE, inet_csk(sk)->icsk_ca_state); 3889 3890 nla_put_u64_64bit(stats, TCP_NLA_BYTES_SENT, tp->bytes_sent, 3891 TCP_NLA_PAD); 3892 nla_put_u64_64bit(stats, TCP_NLA_BYTES_RETRANS, tp->bytes_retrans, 3893 TCP_NLA_PAD); 3894 nla_put_u32(stats, TCP_NLA_DSACK_DUPS, tp->dsack_dups); 3895 nla_put_u32(stats, TCP_NLA_REORD_SEEN, tp->reord_seen); 3896 nla_put_u32(stats, TCP_NLA_SRTT, tp->srtt_us >> 3); 3897 nla_put_u16(stats, TCP_NLA_TIMEOUT_REHASH, tp->timeout_rehash); 3898 nla_put_u32(stats, TCP_NLA_BYTES_NOTSENT, 3899 max_t(int, 0, tp->write_seq - tp->snd_nxt)); 3900 nla_put_u64_64bit(stats, TCP_NLA_EDT, orig_skb->skb_mstamp_ns, 3901 TCP_NLA_PAD); 3902 if (ack_skb) 3903 nla_put_u8(stats, TCP_NLA_TTL, 3904 tcp_skb_ttl_or_hop_limit(ack_skb)); 3905 3906 return stats; 3907 } 3908 3909 static int do_tcp_getsockopt(struct sock *sk, int level, 3910 int optname, char __user *optval, int __user *optlen) 3911 { 3912 struct inet_connection_sock *icsk = inet_csk(sk); 3913 struct tcp_sock *tp = tcp_sk(sk); 3914 struct net *net = sock_net(sk); 3915 int val, len; 3916 3917 if (get_user(len, optlen)) 3918 return -EFAULT; 3919 3920 len = min_t(unsigned int, len, sizeof(int)); 3921 3922 if (len < 0) 3923 return -EINVAL; 3924 3925 switch (optname) { 3926 case TCP_MAXSEG: 3927 val = tp->mss_cache; 3928 if (!val && ((1 << sk->sk_state) & (TCPF_CLOSE | TCPF_LISTEN))) 3929 val = tp->rx_opt.user_mss; 3930 if (tp->repair) 3931 val = tp->rx_opt.mss_clamp; 3932 break; 3933 case TCP_NODELAY: 3934 val = !!(tp->nonagle&TCP_NAGLE_OFF); 3935 break; 3936 case TCP_CORK: 3937 val = !!(tp->nonagle&TCP_NAGLE_CORK); 3938 break; 3939 case TCP_KEEPIDLE: 3940 val = keepalive_time_when(tp) / HZ; 3941 break; 3942 case TCP_KEEPINTVL: 3943 val = keepalive_intvl_when(tp) / HZ; 3944 break; 3945 case TCP_KEEPCNT: 3946 val = keepalive_probes(tp); 3947 break; 3948 case TCP_SYNCNT: 3949 val = icsk->icsk_syn_retries ? : net->ipv4.sysctl_tcp_syn_retries; 3950 break; 3951 case TCP_LINGER2: 3952 val = tp->linger2; 3953 if (val >= 0) 3954 val = (val ? : net->ipv4.sysctl_tcp_fin_timeout) / HZ; 3955 break; 3956 case TCP_DEFER_ACCEPT: 3957 val = retrans_to_secs(icsk->icsk_accept_queue.rskq_defer_accept, 3958 TCP_TIMEOUT_INIT / HZ, TCP_RTO_MAX / HZ); 3959 break; 3960 case TCP_WINDOW_CLAMP: 3961 val = tp->window_clamp; 3962 break; 3963 case TCP_INFO: { 3964 struct tcp_info info; 3965 3966 if (get_user(len, optlen)) 3967 return -EFAULT; 3968 3969 tcp_get_info(sk, &info); 3970 3971 len = min_t(unsigned int, len, sizeof(info)); 3972 if (put_user(len, optlen)) 3973 return -EFAULT; 3974 if (copy_to_user(optval, &info, len)) 3975 return -EFAULT; 3976 return 0; 3977 } 3978 case TCP_CC_INFO: { 3979 const struct tcp_congestion_ops *ca_ops; 3980 union tcp_cc_info info; 3981 size_t sz = 0; 3982 int attr; 3983 3984 if (get_user(len, optlen)) 3985 return -EFAULT; 3986 3987 ca_ops = icsk->icsk_ca_ops; 3988 if (ca_ops && ca_ops->get_info) 3989 sz = ca_ops->get_info(sk, ~0U, &attr, &info); 3990 3991 len = min_t(unsigned int, len, sz); 3992 if (put_user(len, optlen)) 3993 return -EFAULT; 3994 if (copy_to_user(optval, &info, len)) 3995 return -EFAULT; 3996 return 0; 3997 } 3998 case TCP_QUICKACK: 3999 val = !inet_csk_in_pingpong_mode(sk); 4000 break; 4001 4002 case TCP_CONGESTION: 4003 if (get_user(len, optlen)) 4004 return -EFAULT; 4005 len = min_t(unsigned int, len, TCP_CA_NAME_MAX); 4006 if (put_user(len, optlen)) 4007 return -EFAULT; 4008 if (copy_to_user(optval, icsk->icsk_ca_ops->name, len)) 4009 return -EFAULT; 4010 return 0; 4011 4012 case TCP_ULP: 4013 if (get_user(len, optlen)) 4014 return -EFAULT; 4015 len = min_t(unsigned int, len, TCP_ULP_NAME_MAX); 4016 if (!icsk->icsk_ulp_ops) { 4017 if (put_user(0, optlen)) 4018 return -EFAULT; 4019 return 0; 4020 } 4021 if (put_user(len, optlen)) 4022 return -EFAULT; 4023 if (copy_to_user(optval, icsk->icsk_ulp_ops->name, len)) 4024 return -EFAULT; 4025 return 0; 4026 4027 case TCP_FASTOPEN_KEY: { 4028 u64 key[TCP_FASTOPEN_KEY_BUF_LENGTH / sizeof(u64)]; 4029 unsigned int key_len; 4030 4031 if (get_user(len, optlen)) 4032 return -EFAULT; 4033 4034 key_len = tcp_fastopen_get_cipher(net, icsk, key) * 4035 TCP_FASTOPEN_KEY_LENGTH; 4036 len = min_t(unsigned int, len, key_len); 4037 if (put_user(len, optlen)) 4038 return -EFAULT; 4039 if (copy_to_user(optval, key, len)) 4040 return -EFAULT; 4041 return 0; 4042 } 4043 case TCP_THIN_LINEAR_TIMEOUTS: 4044 val = tp->thin_lto; 4045 break; 4046 4047 case TCP_THIN_DUPACK: 4048 val = 0; 4049 break; 4050 4051 case TCP_REPAIR: 4052 val = tp->repair; 4053 break; 4054 4055 case TCP_REPAIR_QUEUE: 4056 if (tp->repair) 4057 val = tp->repair_queue; 4058 else 4059 return -EINVAL; 4060 break; 4061 4062 case TCP_REPAIR_WINDOW: { 4063 struct tcp_repair_window opt; 4064 4065 if (get_user(len, optlen)) 4066 return -EFAULT; 4067 4068 if (len != sizeof(opt)) 4069 return -EINVAL; 4070 4071 if (!tp->repair) 4072 return -EPERM; 4073 4074 opt.snd_wl1 = tp->snd_wl1; 4075 opt.snd_wnd = tp->snd_wnd; 4076 opt.max_window = tp->max_window; 4077 opt.rcv_wnd = tp->rcv_wnd; 4078 opt.rcv_wup = tp->rcv_wup; 4079 4080 if (copy_to_user(optval, &opt, len)) 4081 return -EFAULT; 4082 return 0; 4083 } 4084 case TCP_QUEUE_SEQ: 4085 if (tp->repair_queue == TCP_SEND_QUEUE) 4086 val = tp->write_seq; 4087 else if (tp->repair_queue == TCP_RECV_QUEUE) 4088 val = tp->rcv_nxt; 4089 else 4090 return -EINVAL; 4091 break; 4092 4093 case TCP_USER_TIMEOUT: 4094 val = icsk->icsk_user_timeout; 4095 break; 4096 4097 case TCP_FASTOPEN: 4098 val = icsk->icsk_accept_queue.fastopenq.max_qlen; 4099 break; 4100 4101 case TCP_FASTOPEN_CONNECT: 4102 val = tp->fastopen_connect; 4103 break; 4104 4105 case TCP_FASTOPEN_NO_COOKIE: 4106 val = tp->fastopen_no_cookie; 4107 break; 4108 4109 case TCP_TX_DELAY: 4110 val = tp->tcp_tx_delay; 4111 break; 4112 4113 case TCP_TIMESTAMP: 4114 val = tcp_time_stamp_raw() + tp->tsoffset; 4115 break; 4116 case TCP_NOTSENT_LOWAT: 4117 val = tp->notsent_lowat; 4118 break; 4119 case TCP_INQ: 4120 val = tp->recvmsg_inq; 4121 break; 4122 case TCP_SAVE_SYN: 4123 val = tp->save_syn; 4124 break; 4125 case TCP_SAVED_SYN: { 4126 if (get_user(len, optlen)) 4127 return -EFAULT; 4128 4129 lock_sock(sk); 4130 if (tp->saved_syn) { 4131 if (len < tcp_saved_syn_len(tp->saved_syn)) { 4132 if (put_user(tcp_saved_syn_len(tp->saved_syn), 4133 optlen)) { 4134 release_sock(sk); 4135 return -EFAULT; 4136 } 4137 release_sock(sk); 4138 return -EINVAL; 4139 } 4140 len = tcp_saved_syn_len(tp->saved_syn); 4141 if (put_user(len, optlen)) { 4142 release_sock(sk); 4143 return -EFAULT; 4144 } 4145 if (copy_to_user(optval, tp->saved_syn->data, len)) { 4146 release_sock(sk); 4147 return -EFAULT; 4148 } 4149 tcp_saved_syn_free(tp); 4150 release_sock(sk); 4151 } else { 4152 release_sock(sk); 4153 len = 0; 4154 if (put_user(len, optlen)) 4155 return -EFAULT; 4156 } 4157 return 0; 4158 } 4159 #ifdef CONFIG_MMU 4160 case TCP_ZEROCOPY_RECEIVE: { 4161 struct scm_timestamping_internal tss; 4162 struct tcp_zerocopy_receive zc = {}; 4163 int err; 4164 4165 if (get_user(len, optlen)) 4166 return -EFAULT; 4167 if (len < 0 || 4168 len < offsetofend(struct tcp_zerocopy_receive, length)) 4169 return -EINVAL; 4170 if (unlikely(len > sizeof(zc))) { 4171 err = check_zeroed_user(optval + sizeof(zc), 4172 len - sizeof(zc)); 4173 if (err < 1) 4174 return err == 0 ? -EINVAL : err; 4175 len = sizeof(zc); 4176 if (put_user(len, optlen)) 4177 return -EFAULT; 4178 } 4179 if (copy_from_user(&zc, optval, len)) 4180 return -EFAULT; 4181 if (zc.reserved) 4182 return -EINVAL; 4183 if (zc.msg_flags & ~(TCP_VALID_ZC_MSG_FLAGS)) 4184 return -EINVAL; 4185 lock_sock(sk); 4186 err = tcp_zerocopy_receive(sk, &zc, &tss); 4187 err = BPF_CGROUP_RUN_PROG_GETSOCKOPT_KERN(sk, level, optname, 4188 &zc, &len, err); 4189 release_sock(sk); 4190 if (len >= offsetofend(struct tcp_zerocopy_receive, msg_flags)) 4191 goto zerocopy_rcv_cmsg; 4192 switch (len) { 4193 case offsetofend(struct tcp_zerocopy_receive, msg_flags): 4194 goto zerocopy_rcv_cmsg; 4195 case offsetofend(struct tcp_zerocopy_receive, msg_controllen): 4196 case offsetofend(struct tcp_zerocopy_receive, msg_control): 4197 case offsetofend(struct tcp_zerocopy_receive, flags): 4198 case offsetofend(struct tcp_zerocopy_receive, copybuf_len): 4199 case offsetofend(struct tcp_zerocopy_receive, copybuf_address): 4200 case offsetofend(struct tcp_zerocopy_receive, err): 4201 goto zerocopy_rcv_sk_err; 4202 case offsetofend(struct tcp_zerocopy_receive, inq): 4203 goto zerocopy_rcv_inq; 4204 case offsetofend(struct tcp_zerocopy_receive, length): 4205 default: 4206 goto zerocopy_rcv_out; 4207 } 4208 zerocopy_rcv_cmsg: 4209 if (zc.msg_flags & TCP_CMSG_TS) 4210 tcp_zc_finalize_rx_tstamp(sk, &zc, &tss); 4211 else 4212 zc.msg_flags = 0; 4213 zerocopy_rcv_sk_err: 4214 if (!err) 4215 zc.err = sock_error(sk); 4216 zerocopy_rcv_inq: 4217 zc.inq = tcp_inq_hint(sk); 4218 zerocopy_rcv_out: 4219 if (!err && copy_to_user(optval, &zc, len)) 4220 err = -EFAULT; 4221 return err; 4222 } 4223 #endif 4224 default: 4225 return -ENOPROTOOPT; 4226 } 4227 4228 if (put_user(len, optlen)) 4229 return -EFAULT; 4230 if (copy_to_user(optval, &val, len)) 4231 return -EFAULT; 4232 return 0; 4233 } 4234 4235 bool tcp_bpf_bypass_getsockopt(int level, int optname) 4236 { 4237 /* TCP do_tcp_getsockopt has optimized getsockopt implementation 4238 * to avoid extra socket lock for TCP_ZEROCOPY_RECEIVE. 4239 */ 4240 if (level == SOL_TCP && optname == TCP_ZEROCOPY_RECEIVE) 4241 return true; 4242 4243 return false; 4244 } 4245 EXPORT_SYMBOL(tcp_bpf_bypass_getsockopt); 4246 4247 int tcp_getsockopt(struct sock *sk, int level, int optname, char __user *optval, 4248 int __user *optlen) 4249 { 4250 struct inet_connection_sock *icsk = inet_csk(sk); 4251 4252 if (level != SOL_TCP) 4253 return icsk->icsk_af_ops->getsockopt(sk, level, optname, 4254 optval, optlen); 4255 return do_tcp_getsockopt(sk, level, optname, optval, optlen); 4256 } 4257 EXPORT_SYMBOL(tcp_getsockopt); 4258 4259 #ifdef CONFIG_TCP_MD5SIG 4260 static DEFINE_PER_CPU(struct tcp_md5sig_pool, tcp_md5sig_pool); 4261 static DEFINE_MUTEX(tcp_md5sig_mutex); 4262 static bool tcp_md5sig_pool_populated = false; 4263 4264 static void __tcp_alloc_md5sig_pool(void) 4265 { 4266 struct crypto_ahash *hash; 4267 int cpu; 4268 4269 hash = crypto_alloc_ahash("md5", 0, CRYPTO_ALG_ASYNC); 4270 if (IS_ERR(hash)) 4271 return; 4272 4273 for_each_possible_cpu(cpu) { 4274 void *scratch = per_cpu(tcp_md5sig_pool, cpu).scratch; 4275 struct ahash_request *req; 4276 4277 if (!scratch) { 4278 scratch = kmalloc_node(sizeof(union tcp_md5sum_block) + 4279 sizeof(struct tcphdr), 4280 GFP_KERNEL, 4281 cpu_to_node(cpu)); 4282 if (!scratch) 4283 return; 4284 per_cpu(tcp_md5sig_pool, cpu).scratch = scratch; 4285 } 4286 if (per_cpu(tcp_md5sig_pool, cpu).md5_req) 4287 continue; 4288 4289 req = ahash_request_alloc(hash, GFP_KERNEL); 4290 if (!req) 4291 return; 4292 4293 ahash_request_set_callback(req, 0, NULL, NULL); 4294 4295 per_cpu(tcp_md5sig_pool, cpu).md5_req = req; 4296 } 4297 /* before setting tcp_md5sig_pool_populated, we must commit all writes 4298 * to memory. See smp_rmb() in tcp_get_md5sig_pool() 4299 */ 4300 smp_wmb(); 4301 tcp_md5sig_pool_populated = true; 4302 } 4303 4304 bool tcp_alloc_md5sig_pool(void) 4305 { 4306 if (unlikely(!tcp_md5sig_pool_populated)) { 4307 mutex_lock(&tcp_md5sig_mutex); 4308 4309 if (!tcp_md5sig_pool_populated) { 4310 __tcp_alloc_md5sig_pool(); 4311 if (tcp_md5sig_pool_populated) 4312 static_branch_inc(&tcp_md5_needed); 4313 } 4314 4315 mutex_unlock(&tcp_md5sig_mutex); 4316 } 4317 return tcp_md5sig_pool_populated; 4318 } 4319 EXPORT_SYMBOL(tcp_alloc_md5sig_pool); 4320 4321 4322 /** 4323 * tcp_get_md5sig_pool - get md5sig_pool for this user 4324 * 4325 * We use percpu structure, so if we succeed, we exit with preemption 4326 * and BH disabled, to make sure another thread or softirq handling 4327 * wont try to get same context. 4328 */ 4329 struct tcp_md5sig_pool *tcp_get_md5sig_pool(void) 4330 { 4331 local_bh_disable(); 4332 4333 if (tcp_md5sig_pool_populated) { 4334 /* coupled with smp_wmb() in __tcp_alloc_md5sig_pool() */ 4335 smp_rmb(); 4336 return this_cpu_ptr(&tcp_md5sig_pool); 4337 } 4338 local_bh_enable(); 4339 return NULL; 4340 } 4341 EXPORT_SYMBOL(tcp_get_md5sig_pool); 4342 4343 int tcp_md5_hash_skb_data(struct tcp_md5sig_pool *hp, 4344 const struct sk_buff *skb, unsigned int header_len) 4345 { 4346 struct scatterlist sg; 4347 const struct tcphdr *tp = tcp_hdr(skb); 4348 struct ahash_request *req = hp->md5_req; 4349 unsigned int i; 4350 const unsigned int head_data_len = skb_headlen(skb) > header_len ? 4351 skb_headlen(skb) - header_len : 0; 4352 const struct skb_shared_info *shi = skb_shinfo(skb); 4353 struct sk_buff *frag_iter; 4354 4355 sg_init_table(&sg, 1); 4356 4357 sg_set_buf(&sg, ((u8 *) tp) + header_len, head_data_len); 4358 ahash_request_set_crypt(req, &sg, NULL, head_data_len); 4359 if (crypto_ahash_update(req)) 4360 return 1; 4361 4362 for (i = 0; i < shi->nr_frags; ++i) { 4363 const skb_frag_t *f = &shi->frags[i]; 4364 unsigned int offset = skb_frag_off(f); 4365 struct page *page = skb_frag_page(f) + (offset >> PAGE_SHIFT); 4366 4367 sg_set_page(&sg, page, skb_frag_size(f), 4368 offset_in_page(offset)); 4369 ahash_request_set_crypt(req, &sg, NULL, skb_frag_size(f)); 4370 if (crypto_ahash_update(req)) 4371 return 1; 4372 } 4373 4374 skb_walk_frags(skb, frag_iter) 4375 if (tcp_md5_hash_skb_data(hp, frag_iter, 0)) 4376 return 1; 4377 4378 return 0; 4379 } 4380 EXPORT_SYMBOL(tcp_md5_hash_skb_data); 4381 4382 int tcp_md5_hash_key(struct tcp_md5sig_pool *hp, const struct tcp_md5sig_key *key) 4383 { 4384 u8 keylen = READ_ONCE(key->keylen); /* paired with WRITE_ONCE() in tcp_md5_do_add */ 4385 struct scatterlist sg; 4386 4387 sg_init_one(&sg, key->key, keylen); 4388 ahash_request_set_crypt(hp->md5_req, &sg, NULL, keylen); 4389 4390 /* We use data_race() because tcp_md5_do_add() might change key->key under us */ 4391 return data_race(crypto_ahash_update(hp->md5_req)); 4392 } 4393 EXPORT_SYMBOL(tcp_md5_hash_key); 4394 4395 #endif 4396 4397 void tcp_done(struct sock *sk) 4398 { 4399 struct request_sock *req; 4400 4401 /* We might be called with a new socket, after 4402 * inet_csk_prepare_forced_close() has been called 4403 * so we can not use lockdep_sock_is_held(sk) 4404 */ 4405 req = rcu_dereference_protected(tcp_sk(sk)->fastopen_rsk, 1); 4406 4407 if (sk->sk_state == TCP_SYN_SENT || sk->sk_state == TCP_SYN_RECV) 4408 TCP_INC_STATS(sock_net(sk), TCP_MIB_ATTEMPTFAILS); 4409 4410 tcp_set_state(sk, TCP_CLOSE); 4411 tcp_clear_xmit_timers(sk); 4412 if (req) 4413 reqsk_fastopen_remove(sk, req, false); 4414 4415 sk->sk_shutdown = SHUTDOWN_MASK; 4416 4417 if (!sock_flag(sk, SOCK_DEAD)) 4418 sk->sk_state_change(sk); 4419 else 4420 inet_csk_destroy_sock(sk); 4421 } 4422 EXPORT_SYMBOL_GPL(tcp_done); 4423 4424 int tcp_abort(struct sock *sk, int err) 4425 { 4426 if (!sk_fullsock(sk)) { 4427 if (sk->sk_state == TCP_NEW_SYN_RECV) { 4428 struct request_sock *req = inet_reqsk(sk); 4429 4430 local_bh_disable(); 4431 inet_csk_reqsk_queue_drop(req->rsk_listener, req); 4432 local_bh_enable(); 4433 return 0; 4434 } 4435 return -EOPNOTSUPP; 4436 } 4437 4438 /* Don't race with userspace socket closes such as tcp_close. */ 4439 lock_sock(sk); 4440 4441 if (sk->sk_state == TCP_LISTEN) { 4442 tcp_set_state(sk, TCP_CLOSE); 4443 inet_csk_listen_stop(sk); 4444 } 4445 4446 /* Don't race with BH socket closes such as inet_csk_listen_stop. */ 4447 local_bh_disable(); 4448 bh_lock_sock(sk); 4449 4450 if (!sock_flag(sk, SOCK_DEAD)) { 4451 sk->sk_err = err; 4452 /* This barrier is coupled with smp_rmb() in tcp_poll() */ 4453 smp_wmb(); 4454 sk_error_report(sk); 4455 if (tcp_need_reset(sk->sk_state)) 4456 tcp_send_active_reset(sk, GFP_ATOMIC); 4457 tcp_done(sk); 4458 } 4459 4460 bh_unlock_sock(sk); 4461 local_bh_enable(); 4462 tcp_write_queue_purge(sk); 4463 release_sock(sk); 4464 return 0; 4465 } 4466 EXPORT_SYMBOL_GPL(tcp_abort); 4467 4468 extern struct tcp_congestion_ops tcp_reno; 4469 4470 static __initdata unsigned long thash_entries; 4471 static int __init set_thash_entries(char *str) 4472 { 4473 ssize_t ret; 4474 4475 if (!str) 4476 return 0; 4477 4478 ret = kstrtoul(str, 0, &thash_entries); 4479 if (ret) 4480 return 0; 4481 4482 return 1; 4483 } 4484 __setup("thash_entries=", set_thash_entries); 4485 4486 static void __init tcp_init_mem(void) 4487 { 4488 unsigned long limit = nr_free_buffer_pages() / 16; 4489 4490 limit = max(limit, 128UL); 4491 sysctl_tcp_mem[0] = limit / 4 * 3; /* 4.68 % */ 4492 sysctl_tcp_mem[1] = limit; /* 6.25 % */ 4493 sysctl_tcp_mem[2] = sysctl_tcp_mem[0] * 2; /* 9.37 % */ 4494 } 4495 4496 void __init tcp_init(void) 4497 { 4498 int max_rshare, max_wshare, cnt; 4499 unsigned long limit; 4500 unsigned int i; 4501 4502 BUILD_BUG_ON(TCP_MIN_SND_MSS <= MAX_TCP_OPTION_SPACE); 4503 BUILD_BUG_ON(sizeof(struct tcp_skb_cb) > 4504 sizeof_field(struct sk_buff, cb)); 4505 4506 percpu_counter_init(&tcp_sockets_allocated, 0, GFP_KERNEL); 4507 percpu_counter_init(&tcp_orphan_count, 0, GFP_KERNEL); 4508 inet_hashinfo_init(&tcp_hashinfo); 4509 inet_hashinfo2_init(&tcp_hashinfo, "tcp_listen_portaddr_hash", 4510 thash_entries, 21, /* one slot per 2 MB*/ 4511 0, 64 * 1024); 4512 tcp_hashinfo.bind_bucket_cachep = 4513 kmem_cache_create("tcp_bind_bucket", 4514 sizeof(struct inet_bind_bucket), 0, 4515 SLAB_HWCACHE_ALIGN | SLAB_PANIC | 4516 SLAB_ACCOUNT, 4517 NULL); 4518 4519 /* Size and allocate the main established and bind bucket 4520 * hash tables. 4521 * 4522 * The methodology is similar to that of the buffer cache. 4523 */ 4524 tcp_hashinfo.ehash = 4525 alloc_large_system_hash("TCP established", 4526 sizeof(struct inet_ehash_bucket), 4527 thash_entries, 4528 17, /* one slot per 128 KB of memory */ 4529 0, 4530 NULL, 4531 &tcp_hashinfo.ehash_mask, 4532 0, 4533 thash_entries ? 0 : 512 * 1024); 4534 for (i = 0; i <= tcp_hashinfo.ehash_mask; i++) 4535 INIT_HLIST_NULLS_HEAD(&tcp_hashinfo.ehash[i].chain, i); 4536 4537 if (inet_ehash_locks_alloc(&tcp_hashinfo)) 4538 panic("TCP: failed to alloc ehash_locks"); 4539 tcp_hashinfo.bhash = 4540 alloc_large_system_hash("TCP bind", 4541 sizeof(struct inet_bind_hashbucket), 4542 tcp_hashinfo.ehash_mask + 1, 4543 17, /* one slot per 128 KB of memory */ 4544 0, 4545 &tcp_hashinfo.bhash_size, 4546 NULL, 4547 0, 4548 64 * 1024); 4549 tcp_hashinfo.bhash_size = 1U << tcp_hashinfo.bhash_size; 4550 for (i = 0; i < tcp_hashinfo.bhash_size; i++) { 4551 spin_lock_init(&tcp_hashinfo.bhash[i].lock); 4552 INIT_HLIST_HEAD(&tcp_hashinfo.bhash[i].chain); 4553 } 4554 4555 4556 cnt = tcp_hashinfo.ehash_mask + 1; 4557 sysctl_tcp_max_orphans = cnt / 2; 4558 4559 tcp_init_mem(); 4560 /* Set per-socket limits to no more than 1/128 the pressure threshold */ 4561 limit = nr_free_buffer_pages() << (PAGE_SHIFT - 7); 4562 max_wshare = min(4UL*1024*1024, limit); 4563 max_rshare = min(6UL*1024*1024, limit); 4564 4565 init_net.ipv4.sysctl_tcp_wmem[0] = SK_MEM_QUANTUM; 4566 init_net.ipv4.sysctl_tcp_wmem[1] = 16*1024; 4567 init_net.ipv4.sysctl_tcp_wmem[2] = max(64*1024, max_wshare); 4568 4569 init_net.ipv4.sysctl_tcp_rmem[0] = SK_MEM_QUANTUM; 4570 init_net.ipv4.sysctl_tcp_rmem[1] = 131072; 4571 init_net.ipv4.sysctl_tcp_rmem[2] = max(131072, max_rshare); 4572 4573 pr_info("Hash tables configured (established %u bind %u)\n", 4574 tcp_hashinfo.ehash_mask + 1, tcp_hashinfo.bhash_size); 4575 4576 tcp_v4_init(); 4577 tcp_metrics_init(); 4578 BUG_ON(tcp_register_congestion_control(&tcp_reno) != 0); 4579 tcp_tasklet_init(); 4580 mptcp_init(); 4581 } 4582