1 /* 2 * INET An implementation of the TCP/IP protocol suite for the LINUX 3 * operating system. INET is implemented using the BSD Socket 4 * interface as the means of communication with the user level. 5 * 6 * Implementation of the Transmission Control Protocol(TCP). 7 * 8 * Authors: Ross Biro 9 * Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG> 10 * Mark Evans, <evansmp@uhura.aston.ac.uk> 11 * Corey Minyard <wf-rch!minyard@relay.EU.net> 12 * Florian La Roche, <flla@stud.uni-sb.de> 13 * Charles Hedrick, <hedrick@klinzhai.rutgers.edu> 14 * Linus Torvalds, <torvalds@cs.helsinki.fi> 15 * Alan Cox, <gw4pts@gw4pts.ampr.org> 16 * Matthew Dillon, <dillon@apollo.west.oic.com> 17 * Arnt Gulbrandsen, <agulbra@nvg.unit.no> 18 * Jorge Cwik, <jorge@laser.satlink.net> 19 */ 20 21 /* 22 * Changes: 23 * Pedro Roque : Fast Retransmit/Recovery. 24 * Two receive queues. 25 * Retransmit queue handled by TCP. 26 * Better retransmit timer handling. 27 * New congestion avoidance. 28 * Header prediction. 29 * Variable renaming. 30 * 31 * Eric : Fast Retransmit. 32 * Randy Scott : MSS option defines. 33 * Eric Schenk : Fixes to slow start algorithm. 34 * Eric Schenk : Yet another double ACK bug. 35 * Eric Schenk : Delayed ACK bug fixes. 36 * Eric Schenk : Floyd style fast retrans war avoidance. 37 * David S. Miller : Don't allow zero congestion window. 38 * Eric Schenk : Fix retransmitter so that it sends 39 * next packet on ack of previous packet. 40 * Andi Kleen : Moved open_request checking here 41 * and process RSTs for open_requests. 42 * Andi Kleen : Better prune_queue, and other fixes. 43 * Andrey Savochkin: Fix RTT measurements in the presence of 44 * timestamps. 45 * Andrey Savochkin: Check sequence numbers correctly when 46 * removing SACKs due to in sequence incoming 47 * data segments. 48 * Andi Kleen: Make sure we never ack data there is not 49 * enough room for. Also make this condition 50 * a fatal error if it might still happen. 51 * Andi Kleen: Add tcp_measure_rcv_mss to make 52 * connections with MSS<min(MTU,ann. MSS) 53 * work without delayed acks. 54 * Andi Kleen: Process packets with PSH set in the 55 * fast path. 56 * J Hadi Salim: ECN support 57 * Andrei Gurtov, 58 * Pasi Sarolahti, 59 * Panu Kuhlberg: Experimental audit of TCP (re)transmission 60 * engine. Lots of bugs are found. 61 * Pasi Sarolahti: F-RTO for dealing with spurious RTOs 62 */ 63 64 #define pr_fmt(fmt) "TCP: " fmt 65 66 #include <linux/mm.h> 67 #include <linux/slab.h> 68 #include <linux/module.h> 69 #include <linux/sysctl.h> 70 #include <linux/kernel.h> 71 #include <linux/prefetch.h> 72 #include <net/dst.h> 73 #include <net/tcp.h> 74 #include <net/inet_common.h> 75 #include <linux/ipsec.h> 76 #include <asm/unaligned.h> 77 #include <linux/errqueue.h> 78 79 int sysctl_tcp_timestamps __read_mostly = 1; 80 int sysctl_tcp_window_scaling __read_mostly = 1; 81 int sysctl_tcp_sack __read_mostly = 1; 82 int sysctl_tcp_fack __read_mostly = 1; 83 int sysctl_tcp_reordering __read_mostly = TCP_FASTRETRANS_THRESH; 84 int sysctl_tcp_max_reordering __read_mostly = 300; 85 EXPORT_SYMBOL(sysctl_tcp_reordering); 86 int sysctl_tcp_dsack __read_mostly = 1; 87 int sysctl_tcp_app_win __read_mostly = 31; 88 int sysctl_tcp_adv_win_scale __read_mostly = 1; 89 EXPORT_SYMBOL(sysctl_tcp_adv_win_scale); 90 91 /* rfc5961 challenge ack rate limiting */ 92 int sysctl_tcp_challenge_ack_limit = 100; 93 94 int sysctl_tcp_stdurg __read_mostly; 95 int sysctl_tcp_rfc1337 __read_mostly; 96 int sysctl_tcp_max_orphans __read_mostly = NR_FILE; 97 int sysctl_tcp_frto __read_mostly = 2; 98 int sysctl_tcp_min_rtt_wlen __read_mostly = 300; 99 100 int sysctl_tcp_thin_dupack __read_mostly; 101 102 int sysctl_tcp_moderate_rcvbuf __read_mostly = 1; 103 int sysctl_tcp_early_retrans __read_mostly = 3; 104 int sysctl_tcp_invalid_ratelimit __read_mostly = HZ/2; 105 106 #define FLAG_DATA 0x01 /* Incoming frame contained data. */ 107 #define FLAG_WIN_UPDATE 0x02 /* Incoming ACK was a window update. */ 108 #define FLAG_DATA_ACKED 0x04 /* This ACK acknowledged new data. */ 109 #define FLAG_RETRANS_DATA_ACKED 0x08 /* "" "" some of which was retransmitted. */ 110 #define FLAG_SYN_ACKED 0x10 /* This ACK acknowledged SYN. */ 111 #define FLAG_DATA_SACKED 0x20 /* New SACK. */ 112 #define FLAG_ECE 0x40 /* ECE in this ACK */ 113 #define FLAG_LOST_RETRANS 0x80 /* This ACK marks some retransmission lost */ 114 #define FLAG_SLOWPATH 0x100 /* Do not skip RFC checks for window update.*/ 115 #define FLAG_ORIG_SACK_ACKED 0x200 /* Never retransmitted data are (s)acked */ 116 #define FLAG_SND_UNA_ADVANCED 0x400 /* Snd_una was changed (!= FLAG_DATA_ACKED) */ 117 #define FLAG_DSACKING_ACK 0x800 /* SACK blocks contained D-SACK info */ 118 #define FLAG_SACK_RENEGING 0x2000 /* snd_una advanced to a sacked seq */ 119 #define FLAG_UPDATE_TS_RECENT 0x4000 /* tcp_replace_ts_recent() */ 120 121 #define FLAG_ACKED (FLAG_DATA_ACKED|FLAG_SYN_ACKED) 122 #define FLAG_NOT_DUP (FLAG_DATA|FLAG_WIN_UPDATE|FLAG_ACKED) 123 #define FLAG_CA_ALERT (FLAG_DATA_SACKED|FLAG_ECE) 124 #define FLAG_FORWARD_PROGRESS (FLAG_ACKED|FLAG_DATA_SACKED) 125 126 #define TCP_REMNANT (TCP_FLAG_FIN|TCP_FLAG_URG|TCP_FLAG_SYN|TCP_FLAG_PSH) 127 #define TCP_HP_BITS (~(TCP_RESERVED_BITS|TCP_FLAG_PSH)) 128 129 /* Adapt the MSS value used to make delayed ack decision to the 130 * real world. 131 */ 132 static void tcp_measure_rcv_mss(struct sock *sk, const struct sk_buff *skb) 133 { 134 struct inet_connection_sock *icsk = inet_csk(sk); 135 const unsigned int lss = icsk->icsk_ack.last_seg_size; 136 unsigned int len; 137 138 icsk->icsk_ack.last_seg_size = 0; 139 140 /* skb->len may jitter because of SACKs, even if peer 141 * sends good full-sized frames. 142 */ 143 len = skb_shinfo(skb)->gso_size ? : skb->len; 144 if (len >= icsk->icsk_ack.rcv_mss) { 145 icsk->icsk_ack.rcv_mss = len; 146 } else { 147 /* Otherwise, we make more careful check taking into account, 148 * that SACKs block is variable. 149 * 150 * "len" is invariant segment length, including TCP header. 151 */ 152 len += skb->data - skb_transport_header(skb); 153 if (len >= TCP_MSS_DEFAULT + sizeof(struct tcphdr) || 154 /* If PSH is not set, packet should be 155 * full sized, provided peer TCP is not badly broken. 156 * This observation (if it is correct 8)) allows 157 * to handle super-low mtu links fairly. 158 */ 159 (len >= TCP_MIN_MSS + sizeof(struct tcphdr) && 160 !(tcp_flag_word(tcp_hdr(skb)) & TCP_REMNANT))) { 161 /* Subtract also invariant (if peer is RFC compliant), 162 * tcp header plus fixed timestamp option length. 163 * Resulting "len" is MSS free of SACK jitter. 164 */ 165 len -= tcp_sk(sk)->tcp_header_len; 166 icsk->icsk_ack.last_seg_size = len; 167 if (len == lss) { 168 icsk->icsk_ack.rcv_mss = len; 169 return; 170 } 171 } 172 if (icsk->icsk_ack.pending & ICSK_ACK_PUSHED) 173 icsk->icsk_ack.pending |= ICSK_ACK_PUSHED2; 174 icsk->icsk_ack.pending |= ICSK_ACK_PUSHED; 175 } 176 } 177 178 static void tcp_incr_quickack(struct sock *sk) 179 { 180 struct inet_connection_sock *icsk = inet_csk(sk); 181 unsigned int quickacks = tcp_sk(sk)->rcv_wnd / (2 * icsk->icsk_ack.rcv_mss); 182 183 if (quickacks == 0) 184 quickacks = 2; 185 if (quickacks > icsk->icsk_ack.quick) 186 icsk->icsk_ack.quick = min(quickacks, TCP_MAX_QUICKACKS); 187 } 188 189 static void tcp_enter_quickack_mode(struct sock *sk) 190 { 191 struct inet_connection_sock *icsk = inet_csk(sk); 192 tcp_incr_quickack(sk); 193 icsk->icsk_ack.pingpong = 0; 194 icsk->icsk_ack.ato = TCP_ATO_MIN; 195 } 196 197 /* Send ACKs quickly, if "quick" count is not exhausted 198 * and the session is not interactive. 199 */ 200 201 static bool tcp_in_quickack_mode(struct sock *sk) 202 { 203 const struct inet_connection_sock *icsk = inet_csk(sk); 204 const struct dst_entry *dst = __sk_dst_get(sk); 205 206 return (dst && dst_metric(dst, RTAX_QUICKACK)) || 207 (icsk->icsk_ack.quick && !icsk->icsk_ack.pingpong); 208 } 209 210 static void tcp_ecn_queue_cwr(struct tcp_sock *tp) 211 { 212 if (tp->ecn_flags & TCP_ECN_OK) 213 tp->ecn_flags |= TCP_ECN_QUEUE_CWR; 214 } 215 216 static void tcp_ecn_accept_cwr(struct tcp_sock *tp, const struct sk_buff *skb) 217 { 218 if (tcp_hdr(skb)->cwr) 219 tp->ecn_flags &= ~TCP_ECN_DEMAND_CWR; 220 } 221 222 static void tcp_ecn_withdraw_cwr(struct tcp_sock *tp) 223 { 224 tp->ecn_flags &= ~TCP_ECN_DEMAND_CWR; 225 } 226 227 static void __tcp_ecn_check_ce(struct tcp_sock *tp, const struct sk_buff *skb) 228 { 229 switch (TCP_SKB_CB(skb)->ip_dsfield & INET_ECN_MASK) { 230 case INET_ECN_NOT_ECT: 231 /* Funny extension: if ECT is not set on a segment, 232 * and we already seen ECT on a previous segment, 233 * it is probably a retransmit. 234 */ 235 if (tp->ecn_flags & TCP_ECN_SEEN) 236 tcp_enter_quickack_mode((struct sock *)tp); 237 break; 238 case INET_ECN_CE: 239 if (tcp_ca_needs_ecn((struct sock *)tp)) 240 tcp_ca_event((struct sock *)tp, CA_EVENT_ECN_IS_CE); 241 242 if (!(tp->ecn_flags & TCP_ECN_DEMAND_CWR)) { 243 /* Better not delay acks, sender can have a very low cwnd */ 244 tcp_enter_quickack_mode((struct sock *)tp); 245 tp->ecn_flags |= TCP_ECN_DEMAND_CWR; 246 } 247 tp->ecn_flags |= TCP_ECN_SEEN; 248 break; 249 default: 250 if (tcp_ca_needs_ecn((struct sock *)tp)) 251 tcp_ca_event((struct sock *)tp, CA_EVENT_ECN_NO_CE); 252 tp->ecn_flags |= TCP_ECN_SEEN; 253 break; 254 } 255 } 256 257 static void tcp_ecn_check_ce(struct tcp_sock *tp, const struct sk_buff *skb) 258 { 259 if (tp->ecn_flags & TCP_ECN_OK) 260 __tcp_ecn_check_ce(tp, skb); 261 } 262 263 static void tcp_ecn_rcv_synack(struct tcp_sock *tp, const struct tcphdr *th) 264 { 265 if ((tp->ecn_flags & TCP_ECN_OK) && (!th->ece || th->cwr)) 266 tp->ecn_flags &= ~TCP_ECN_OK; 267 } 268 269 static void tcp_ecn_rcv_syn(struct tcp_sock *tp, const struct tcphdr *th) 270 { 271 if ((tp->ecn_flags & TCP_ECN_OK) && (!th->ece || !th->cwr)) 272 tp->ecn_flags &= ~TCP_ECN_OK; 273 } 274 275 static bool tcp_ecn_rcv_ecn_echo(const struct tcp_sock *tp, const struct tcphdr *th) 276 { 277 if (th->ece && !th->syn && (tp->ecn_flags & TCP_ECN_OK)) 278 return true; 279 return false; 280 } 281 282 /* Buffer size and advertised window tuning. 283 * 284 * 1. Tuning sk->sk_sndbuf, when connection enters established state. 285 */ 286 287 static void tcp_sndbuf_expand(struct sock *sk) 288 { 289 const struct tcp_sock *tp = tcp_sk(sk); 290 int sndmem, per_mss; 291 u32 nr_segs; 292 293 /* Worst case is non GSO/TSO : each frame consumes one skb 294 * and skb->head is kmalloced using power of two area of memory 295 */ 296 per_mss = max_t(u32, tp->rx_opt.mss_clamp, tp->mss_cache) + 297 MAX_TCP_HEADER + 298 SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); 299 300 per_mss = roundup_pow_of_two(per_mss) + 301 SKB_DATA_ALIGN(sizeof(struct sk_buff)); 302 303 nr_segs = max_t(u32, TCP_INIT_CWND, tp->snd_cwnd); 304 nr_segs = max_t(u32, nr_segs, tp->reordering + 1); 305 306 /* Fast Recovery (RFC 5681 3.2) : 307 * Cubic needs 1.7 factor, rounded to 2 to include 308 * extra cushion (application might react slowly to POLLOUT) 309 */ 310 sndmem = 2 * nr_segs * per_mss; 311 312 if (sk->sk_sndbuf < sndmem) 313 sk->sk_sndbuf = min(sndmem, sysctl_tcp_wmem[2]); 314 } 315 316 /* 2. Tuning advertised window (window_clamp, rcv_ssthresh) 317 * 318 * All tcp_full_space() is split to two parts: "network" buffer, allocated 319 * forward and advertised in receiver window (tp->rcv_wnd) and 320 * "application buffer", required to isolate scheduling/application 321 * latencies from network. 322 * window_clamp is maximal advertised window. It can be less than 323 * tcp_full_space(), in this case tcp_full_space() - window_clamp 324 * is reserved for "application" buffer. The less window_clamp is 325 * the smoother our behaviour from viewpoint of network, but the lower 326 * throughput and the higher sensitivity of the connection to losses. 8) 327 * 328 * rcv_ssthresh is more strict window_clamp used at "slow start" 329 * phase to predict further behaviour of this connection. 330 * It is used for two goals: 331 * - to enforce header prediction at sender, even when application 332 * requires some significant "application buffer". It is check #1. 333 * - to prevent pruning of receive queue because of misprediction 334 * of receiver window. Check #2. 335 * 336 * The scheme does not work when sender sends good segments opening 337 * window and then starts to feed us spaghetti. But it should work 338 * in common situations. Otherwise, we have to rely on queue collapsing. 339 */ 340 341 /* Slow part of check#2. */ 342 static int __tcp_grow_window(const struct sock *sk, const struct sk_buff *skb) 343 { 344 struct tcp_sock *tp = tcp_sk(sk); 345 /* Optimize this! */ 346 int truesize = tcp_win_from_space(skb->truesize) >> 1; 347 int window = tcp_win_from_space(sysctl_tcp_rmem[2]) >> 1; 348 349 while (tp->rcv_ssthresh <= window) { 350 if (truesize <= skb->len) 351 return 2 * inet_csk(sk)->icsk_ack.rcv_mss; 352 353 truesize >>= 1; 354 window >>= 1; 355 } 356 return 0; 357 } 358 359 static void tcp_grow_window(struct sock *sk, const struct sk_buff *skb) 360 { 361 struct tcp_sock *tp = tcp_sk(sk); 362 363 /* Check #1 */ 364 if (tp->rcv_ssthresh < tp->window_clamp && 365 (int)tp->rcv_ssthresh < tcp_space(sk) && 366 !tcp_under_memory_pressure(sk)) { 367 int incr; 368 369 /* Check #2. Increase window, if skb with such overhead 370 * will fit to rcvbuf in future. 371 */ 372 if (tcp_win_from_space(skb->truesize) <= skb->len) 373 incr = 2 * tp->advmss; 374 else 375 incr = __tcp_grow_window(sk, skb); 376 377 if (incr) { 378 incr = max_t(int, incr, 2 * skb->len); 379 tp->rcv_ssthresh = min(tp->rcv_ssthresh + incr, 380 tp->window_clamp); 381 inet_csk(sk)->icsk_ack.quick |= 1; 382 } 383 } 384 } 385 386 /* 3. Tuning rcvbuf, when connection enters established state. */ 387 static void tcp_fixup_rcvbuf(struct sock *sk) 388 { 389 u32 mss = tcp_sk(sk)->advmss; 390 int rcvmem; 391 392 rcvmem = 2 * SKB_TRUESIZE(mss + MAX_TCP_HEADER) * 393 tcp_default_init_rwnd(mss); 394 395 /* Dynamic Right Sizing (DRS) has 2 to 3 RTT latency 396 * Allow enough cushion so that sender is not limited by our window 397 */ 398 if (sysctl_tcp_moderate_rcvbuf) 399 rcvmem <<= 2; 400 401 if (sk->sk_rcvbuf < rcvmem) 402 sk->sk_rcvbuf = min(rcvmem, sysctl_tcp_rmem[2]); 403 } 404 405 /* 4. Try to fixup all. It is made immediately after connection enters 406 * established state. 407 */ 408 void tcp_init_buffer_space(struct sock *sk) 409 { 410 struct tcp_sock *tp = tcp_sk(sk); 411 int maxwin; 412 413 if (!(sk->sk_userlocks & SOCK_RCVBUF_LOCK)) 414 tcp_fixup_rcvbuf(sk); 415 if (!(sk->sk_userlocks & SOCK_SNDBUF_LOCK)) 416 tcp_sndbuf_expand(sk); 417 418 tp->rcvq_space.space = tp->rcv_wnd; 419 tp->rcvq_space.time = tcp_time_stamp; 420 tp->rcvq_space.seq = tp->copied_seq; 421 422 maxwin = tcp_full_space(sk); 423 424 if (tp->window_clamp >= maxwin) { 425 tp->window_clamp = maxwin; 426 427 if (sysctl_tcp_app_win && maxwin > 4 * tp->advmss) 428 tp->window_clamp = max(maxwin - 429 (maxwin >> sysctl_tcp_app_win), 430 4 * tp->advmss); 431 } 432 433 /* Force reservation of one segment. */ 434 if (sysctl_tcp_app_win && 435 tp->window_clamp > 2 * tp->advmss && 436 tp->window_clamp + tp->advmss > maxwin) 437 tp->window_clamp = max(2 * tp->advmss, maxwin - tp->advmss); 438 439 tp->rcv_ssthresh = min(tp->rcv_ssthresh, tp->window_clamp); 440 tp->snd_cwnd_stamp = tcp_time_stamp; 441 } 442 443 /* 5. Recalculate window clamp after socket hit its memory bounds. */ 444 static void tcp_clamp_window(struct sock *sk) 445 { 446 struct tcp_sock *tp = tcp_sk(sk); 447 struct inet_connection_sock *icsk = inet_csk(sk); 448 449 icsk->icsk_ack.quick = 0; 450 451 if (sk->sk_rcvbuf < sysctl_tcp_rmem[2] && 452 !(sk->sk_userlocks & SOCK_RCVBUF_LOCK) && 453 !tcp_under_memory_pressure(sk) && 454 sk_memory_allocated(sk) < sk_prot_mem_limits(sk, 0)) { 455 sk->sk_rcvbuf = min(atomic_read(&sk->sk_rmem_alloc), 456 sysctl_tcp_rmem[2]); 457 } 458 if (atomic_read(&sk->sk_rmem_alloc) > sk->sk_rcvbuf) 459 tp->rcv_ssthresh = min(tp->window_clamp, 2U * tp->advmss); 460 } 461 462 /* Initialize RCV_MSS value. 463 * RCV_MSS is an our guess about MSS used by the peer. 464 * We haven't any direct information about the MSS. 465 * It's better to underestimate the RCV_MSS rather than overestimate. 466 * Overestimations make us ACKing less frequently than needed. 467 * Underestimations are more easy to detect and fix by tcp_measure_rcv_mss(). 468 */ 469 void tcp_initialize_rcv_mss(struct sock *sk) 470 { 471 const struct tcp_sock *tp = tcp_sk(sk); 472 unsigned int hint = min_t(unsigned int, tp->advmss, tp->mss_cache); 473 474 hint = min(hint, tp->rcv_wnd / 2); 475 hint = min(hint, TCP_MSS_DEFAULT); 476 hint = max(hint, TCP_MIN_MSS); 477 478 inet_csk(sk)->icsk_ack.rcv_mss = hint; 479 } 480 EXPORT_SYMBOL(tcp_initialize_rcv_mss); 481 482 /* Receiver "autotuning" code. 483 * 484 * The algorithm for RTT estimation w/o timestamps is based on 485 * Dynamic Right-Sizing (DRS) by Wu Feng and Mike Fisk of LANL. 486 * <http://public.lanl.gov/radiant/pubs.html#DRS> 487 * 488 * More detail on this code can be found at 489 * <http://staff.psc.edu/jheffner/>, 490 * though this reference is out of date. A new paper 491 * is pending. 492 */ 493 static void tcp_rcv_rtt_update(struct tcp_sock *tp, u32 sample, int win_dep) 494 { 495 u32 new_sample = tp->rcv_rtt_est.rtt; 496 long m = sample; 497 498 if (m == 0) 499 m = 1; 500 501 if (new_sample != 0) { 502 /* If we sample in larger samples in the non-timestamp 503 * case, we could grossly overestimate the RTT especially 504 * with chatty applications or bulk transfer apps which 505 * are stalled on filesystem I/O. 506 * 507 * Also, since we are only going for a minimum in the 508 * non-timestamp case, we do not smooth things out 509 * else with timestamps disabled convergence takes too 510 * long. 511 */ 512 if (!win_dep) { 513 m -= (new_sample >> 3); 514 new_sample += m; 515 } else { 516 m <<= 3; 517 if (m < new_sample) 518 new_sample = m; 519 } 520 } else { 521 /* No previous measure. */ 522 new_sample = m << 3; 523 } 524 525 if (tp->rcv_rtt_est.rtt != new_sample) 526 tp->rcv_rtt_est.rtt = new_sample; 527 } 528 529 static inline void tcp_rcv_rtt_measure(struct tcp_sock *tp) 530 { 531 if (tp->rcv_rtt_est.time == 0) 532 goto new_measure; 533 if (before(tp->rcv_nxt, tp->rcv_rtt_est.seq)) 534 return; 535 tcp_rcv_rtt_update(tp, tcp_time_stamp - tp->rcv_rtt_est.time, 1); 536 537 new_measure: 538 tp->rcv_rtt_est.seq = tp->rcv_nxt + tp->rcv_wnd; 539 tp->rcv_rtt_est.time = tcp_time_stamp; 540 } 541 542 static inline void tcp_rcv_rtt_measure_ts(struct sock *sk, 543 const struct sk_buff *skb) 544 { 545 struct tcp_sock *tp = tcp_sk(sk); 546 if (tp->rx_opt.rcv_tsecr && 547 (TCP_SKB_CB(skb)->end_seq - 548 TCP_SKB_CB(skb)->seq >= inet_csk(sk)->icsk_ack.rcv_mss)) 549 tcp_rcv_rtt_update(tp, tcp_time_stamp - tp->rx_opt.rcv_tsecr, 0); 550 } 551 552 /* 553 * This function should be called every time data is copied to user space. 554 * It calculates the appropriate TCP receive buffer space. 555 */ 556 void tcp_rcv_space_adjust(struct sock *sk) 557 { 558 struct tcp_sock *tp = tcp_sk(sk); 559 int time; 560 int copied; 561 562 time = tcp_time_stamp - tp->rcvq_space.time; 563 if (time < (tp->rcv_rtt_est.rtt >> 3) || tp->rcv_rtt_est.rtt == 0) 564 return; 565 566 /* Number of bytes copied to user in last RTT */ 567 copied = tp->copied_seq - tp->rcvq_space.seq; 568 if (copied <= tp->rcvq_space.space) 569 goto new_measure; 570 571 /* A bit of theory : 572 * copied = bytes received in previous RTT, our base window 573 * To cope with packet losses, we need a 2x factor 574 * To cope with slow start, and sender growing its cwin by 100 % 575 * every RTT, we need a 4x factor, because the ACK we are sending 576 * now is for the next RTT, not the current one : 577 * <prev RTT . ><current RTT .. ><next RTT .... > 578 */ 579 580 if (sysctl_tcp_moderate_rcvbuf && 581 !(sk->sk_userlocks & SOCK_RCVBUF_LOCK)) { 582 int rcvwin, rcvmem, rcvbuf; 583 584 /* minimal window to cope with packet losses, assuming 585 * steady state. Add some cushion because of small variations. 586 */ 587 rcvwin = (copied << 1) + 16 * tp->advmss; 588 589 /* If rate increased by 25%, 590 * assume slow start, rcvwin = 3 * copied 591 * If rate increased by 50%, 592 * assume sender can use 2x growth, rcvwin = 4 * copied 593 */ 594 if (copied >= 595 tp->rcvq_space.space + (tp->rcvq_space.space >> 2)) { 596 if (copied >= 597 tp->rcvq_space.space + (tp->rcvq_space.space >> 1)) 598 rcvwin <<= 1; 599 else 600 rcvwin += (rcvwin >> 1); 601 } 602 603 rcvmem = SKB_TRUESIZE(tp->advmss + MAX_TCP_HEADER); 604 while (tcp_win_from_space(rcvmem) < tp->advmss) 605 rcvmem += 128; 606 607 rcvbuf = min(rcvwin / tp->advmss * rcvmem, sysctl_tcp_rmem[2]); 608 if (rcvbuf > sk->sk_rcvbuf) { 609 sk->sk_rcvbuf = rcvbuf; 610 611 /* Make the window clamp follow along. */ 612 tp->window_clamp = rcvwin; 613 } 614 } 615 tp->rcvq_space.space = copied; 616 617 new_measure: 618 tp->rcvq_space.seq = tp->copied_seq; 619 tp->rcvq_space.time = tcp_time_stamp; 620 } 621 622 /* There is something which you must keep in mind when you analyze the 623 * behavior of the tp->ato delayed ack timeout interval. When a 624 * connection starts up, we want to ack as quickly as possible. The 625 * problem is that "good" TCP's do slow start at the beginning of data 626 * transmission. The means that until we send the first few ACK's the 627 * sender will sit on his end and only queue most of his data, because 628 * he can only send snd_cwnd unacked packets at any given time. For 629 * each ACK we send, he increments snd_cwnd and transmits more of his 630 * queue. -DaveM 631 */ 632 static void tcp_event_data_recv(struct sock *sk, struct sk_buff *skb) 633 { 634 struct tcp_sock *tp = tcp_sk(sk); 635 struct inet_connection_sock *icsk = inet_csk(sk); 636 u32 now; 637 638 inet_csk_schedule_ack(sk); 639 640 tcp_measure_rcv_mss(sk, skb); 641 642 tcp_rcv_rtt_measure(tp); 643 644 now = tcp_time_stamp; 645 646 if (!icsk->icsk_ack.ato) { 647 /* The _first_ data packet received, initialize 648 * delayed ACK engine. 649 */ 650 tcp_incr_quickack(sk); 651 icsk->icsk_ack.ato = TCP_ATO_MIN; 652 } else { 653 int m = now - icsk->icsk_ack.lrcvtime; 654 655 if (m <= TCP_ATO_MIN / 2) { 656 /* The fastest case is the first. */ 657 icsk->icsk_ack.ato = (icsk->icsk_ack.ato >> 1) + TCP_ATO_MIN / 2; 658 } else if (m < icsk->icsk_ack.ato) { 659 icsk->icsk_ack.ato = (icsk->icsk_ack.ato >> 1) + m; 660 if (icsk->icsk_ack.ato > icsk->icsk_rto) 661 icsk->icsk_ack.ato = icsk->icsk_rto; 662 } else if (m > icsk->icsk_rto) { 663 /* Too long gap. Apparently sender failed to 664 * restart window, so that we send ACKs quickly. 665 */ 666 tcp_incr_quickack(sk); 667 sk_mem_reclaim(sk); 668 } 669 } 670 icsk->icsk_ack.lrcvtime = now; 671 672 tcp_ecn_check_ce(tp, skb); 673 674 if (skb->len >= 128) 675 tcp_grow_window(sk, skb); 676 } 677 678 /* Called to compute a smoothed rtt estimate. The data fed to this 679 * routine either comes from timestamps, or from segments that were 680 * known _not_ to have been retransmitted [see Karn/Partridge 681 * Proceedings SIGCOMM 87]. The algorithm is from the SIGCOMM 88 682 * piece by Van Jacobson. 683 * NOTE: the next three routines used to be one big routine. 684 * To save cycles in the RFC 1323 implementation it was better to break 685 * it up into three procedures. -- erics 686 */ 687 static void tcp_rtt_estimator(struct sock *sk, long mrtt_us) 688 { 689 struct tcp_sock *tp = tcp_sk(sk); 690 long m = mrtt_us; /* RTT */ 691 u32 srtt = tp->srtt_us; 692 693 /* The following amusing code comes from Jacobson's 694 * article in SIGCOMM '88. Note that rtt and mdev 695 * are scaled versions of rtt and mean deviation. 696 * This is designed to be as fast as possible 697 * m stands for "measurement". 698 * 699 * On a 1990 paper the rto value is changed to: 700 * RTO = rtt + 4 * mdev 701 * 702 * Funny. This algorithm seems to be very broken. 703 * These formulae increase RTO, when it should be decreased, increase 704 * too slowly, when it should be increased quickly, decrease too quickly 705 * etc. I guess in BSD RTO takes ONE value, so that it is absolutely 706 * does not matter how to _calculate_ it. Seems, it was trap 707 * that VJ failed to avoid. 8) 708 */ 709 if (srtt != 0) { 710 m -= (srtt >> 3); /* m is now error in rtt est */ 711 srtt += m; /* rtt = 7/8 rtt + 1/8 new */ 712 if (m < 0) { 713 m = -m; /* m is now abs(error) */ 714 m -= (tp->mdev_us >> 2); /* similar update on mdev */ 715 /* This is similar to one of Eifel findings. 716 * Eifel blocks mdev updates when rtt decreases. 717 * This solution is a bit different: we use finer gain 718 * for mdev in this case (alpha*beta). 719 * Like Eifel it also prevents growth of rto, 720 * but also it limits too fast rto decreases, 721 * happening in pure Eifel. 722 */ 723 if (m > 0) 724 m >>= 3; 725 } else { 726 m -= (tp->mdev_us >> 2); /* similar update on mdev */ 727 } 728 tp->mdev_us += m; /* mdev = 3/4 mdev + 1/4 new */ 729 if (tp->mdev_us > tp->mdev_max_us) { 730 tp->mdev_max_us = tp->mdev_us; 731 if (tp->mdev_max_us > tp->rttvar_us) 732 tp->rttvar_us = tp->mdev_max_us; 733 } 734 if (after(tp->snd_una, tp->rtt_seq)) { 735 if (tp->mdev_max_us < tp->rttvar_us) 736 tp->rttvar_us -= (tp->rttvar_us - tp->mdev_max_us) >> 2; 737 tp->rtt_seq = tp->snd_nxt; 738 tp->mdev_max_us = tcp_rto_min_us(sk); 739 } 740 } else { 741 /* no previous measure. */ 742 srtt = m << 3; /* take the measured time to be rtt */ 743 tp->mdev_us = m << 1; /* make sure rto = 3*rtt */ 744 tp->rttvar_us = max(tp->mdev_us, tcp_rto_min_us(sk)); 745 tp->mdev_max_us = tp->rttvar_us; 746 tp->rtt_seq = tp->snd_nxt; 747 } 748 tp->srtt_us = max(1U, srtt); 749 } 750 751 /* Set the sk_pacing_rate to allow proper sizing of TSO packets. 752 * Note: TCP stack does not yet implement pacing. 753 * FQ packet scheduler can be used to implement cheap but effective 754 * TCP pacing, to smooth the burst on large writes when packets 755 * in flight is significantly lower than cwnd (or rwin) 756 */ 757 int sysctl_tcp_pacing_ss_ratio __read_mostly = 200; 758 int sysctl_tcp_pacing_ca_ratio __read_mostly = 120; 759 760 static void tcp_update_pacing_rate(struct sock *sk) 761 { 762 const struct tcp_sock *tp = tcp_sk(sk); 763 u64 rate; 764 765 /* set sk_pacing_rate to 200 % of current rate (mss * cwnd / srtt) */ 766 rate = (u64)tp->mss_cache * ((USEC_PER_SEC / 100) << 3); 767 768 /* current rate is (cwnd * mss) / srtt 769 * In Slow Start [1], set sk_pacing_rate to 200 % the current rate. 770 * In Congestion Avoidance phase, set it to 120 % the current rate. 771 * 772 * [1] : Normal Slow Start condition is (tp->snd_cwnd < tp->snd_ssthresh) 773 * If snd_cwnd >= (tp->snd_ssthresh / 2), we are approaching 774 * end of slow start and should slow down. 775 */ 776 if (tp->snd_cwnd < tp->snd_ssthresh / 2) 777 rate *= sysctl_tcp_pacing_ss_ratio; 778 else 779 rate *= sysctl_tcp_pacing_ca_ratio; 780 781 rate *= max(tp->snd_cwnd, tp->packets_out); 782 783 if (likely(tp->srtt_us)) 784 do_div(rate, tp->srtt_us); 785 786 /* ACCESS_ONCE() is needed because sch_fq fetches sk_pacing_rate 787 * without any lock. We want to make sure compiler wont store 788 * intermediate values in this location. 789 */ 790 ACCESS_ONCE(sk->sk_pacing_rate) = min_t(u64, rate, 791 sk->sk_max_pacing_rate); 792 } 793 794 /* Calculate rto without backoff. This is the second half of Van Jacobson's 795 * routine referred to above. 796 */ 797 static void tcp_set_rto(struct sock *sk) 798 { 799 const struct tcp_sock *tp = tcp_sk(sk); 800 /* Old crap is replaced with new one. 8) 801 * 802 * More seriously: 803 * 1. If rtt variance happened to be less 50msec, it is hallucination. 804 * It cannot be less due to utterly erratic ACK generation made 805 * at least by solaris and freebsd. "Erratic ACKs" has _nothing_ 806 * to do with delayed acks, because at cwnd>2 true delack timeout 807 * is invisible. Actually, Linux-2.4 also generates erratic 808 * ACKs in some circumstances. 809 */ 810 inet_csk(sk)->icsk_rto = __tcp_set_rto(tp); 811 812 /* 2. Fixups made earlier cannot be right. 813 * If we do not estimate RTO correctly without them, 814 * all the algo is pure shit and should be replaced 815 * with correct one. It is exactly, which we pretend to do. 816 */ 817 818 /* NOTE: clamping at TCP_RTO_MIN is not required, current algo 819 * guarantees that rto is higher. 820 */ 821 tcp_bound_rto(sk); 822 } 823 824 __u32 tcp_init_cwnd(const struct tcp_sock *tp, const struct dst_entry *dst) 825 { 826 __u32 cwnd = (dst ? dst_metric(dst, RTAX_INITCWND) : 0); 827 828 if (!cwnd) 829 cwnd = TCP_INIT_CWND; 830 return min_t(__u32, cwnd, tp->snd_cwnd_clamp); 831 } 832 833 /* 834 * Packet counting of FACK is based on in-order assumptions, therefore TCP 835 * disables it when reordering is detected 836 */ 837 void tcp_disable_fack(struct tcp_sock *tp) 838 { 839 /* RFC3517 uses different metric in lost marker => reset on change */ 840 if (tcp_is_fack(tp)) 841 tp->lost_skb_hint = NULL; 842 tp->rx_opt.sack_ok &= ~TCP_FACK_ENABLED; 843 } 844 845 /* Take a notice that peer is sending D-SACKs */ 846 static void tcp_dsack_seen(struct tcp_sock *tp) 847 { 848 tp->rx_opt.sack_ok |= TCP_DSACK_SEEN; 849 } 850 851 static void tcp_update_reordering(struct sock *sk, const int metric, 852 const int ts) 853 { 854 struct tcp_sock *tp = tcp_sk(sk); 855 if (metric > tp->reordering) { 856 int mib_idx; 857 858 tp->reordering = min(sysctl_tcp_max_reordering, metric); 859 860 /* This exciting event is worth to be remembered. 8) */ 861 if (ts) 862 mib_idx = LINUX_MIB_TCPTSREORDER; 863 else if (tcp_is_reno(tp)) 864 mib_idx = LINUX_MIB_TCPRENOREORDER; 865 else if (tcp_is_fack(tp)) 866 mib_idx = LINUX_MIB_TCPFACKREORDER; 867 else 868 mib_idx = LINUX_MIB_TCPSACKREORDER; 869 870 NET_INC_STATS_BH(sock_net(sk), mib_idx); 871 #if FASTRETRANS_DEBUG > 1 872 pr_debug("Disorder%d %d %u f%u s%u rr%d\n", 873 tp->rx_opt.sack_ok, inet_csk(sk)->icsk_ca_state, 874 tp->reordering, 875 tp->fackets_out, 876 tp->sacked_out, 877 tp->undo_marker ? tp->undo_retrans : 0); 878 #endif 879 tcp_disable_fack(tp); 880 } 881 882 if (metric > 0) 883 tcp_disable_early_retrans(tp); 884 tp->rack.reord = 1; 885 } 886 887 /* This must be called before lost_out is incremented */ 888 static void tcp_verify_retransmit_hint(struct tcp_sock *tp, struct sk_buff *skb) 889 { 890 if (!tp->retransmit_skb_hint || 891 before(TCP_SKB_CB(skb)->seq, 892 TCP_SKB_CB(tp->retransmit_skb_hint)->seq)) 893 tp->retransmit_skb_hint = skb; 894 895 if (!tp->lost_out || 896 after(TCP_SKB_CB(skb)->end_seq, tp->retransmit_high)) 897 tp->retransmit_high = TCP_SKB_CB(skb)->end_seq; 898 } 899 900 static void tcp_skb_mark_lost(struct tcp_sock *tp, struct sk_buff *skb) 901 { 902 if (!(TCP_SKB_CB(skb)->sacked & (TCPCB_LOST|TCPCB_SACKED_ACKED))) { 903 tcp_verify_retransmit_hint(tp, skb); 904 905 tp->lost_out += tcp_skb_pcount(skb); 906 TCP_SKB_CB(skb)->sacked |= TCPCB_LOST; 907 } 908 } 909 910 void tcp_skb_mark_lost_uncond_verify(struct tcp_sock *tp, struct sk_buff *skb) 911 { 912 tcp_verify_retransmit_hint(tp, skb); 913 914 if (!(TCP_SKB_CB(skb)->sacked & (TCPCB_LOST|TCPCB_SACKED_ACKED))) { 915 tp->lost_out += tcp_skb_pcount(skb); 916 TCP_SKB_CB(skb)->sacked |= TCPCB_LOST; 917 } 918 } 919 920 /* This procedure tags the retransmission queue when SACKs arrive. 921 * 922 * We have three tag bits: SACKED(S), RETRANS(R) and LOST(L). 923 * Packets in queue with these bits set are counted in variables 924 * sacked_out, retrans_out and lost_out, correspondingly. 925 * 926 * Valid combinations are: 927 * Tag InFlight Description 928 * 0 1 - orig segment is in flight. 929 * S 0 - nothing flies, orig reached receiver. 930 * L 0 - nothing flies, orig lost by net. 931 * R 2 - both orig and retransmit are in flight. 932 * L|R 1 - orig is lost, retransmit is in flight. 933 * S|R 1 - orig reached receiver, retrans is still in flight. 934 * (L|S|R is logically valid, it could occur when L|R is sacked, 935 * but it is equivalent to plain S and code short-curcuits it to S. 936 * L|S is logically invalid, it would mean -1 packet in flight 8)) 937 * 938 * These 6 states form finite state machine, controlled by the following events: 939 * 1. New ACK (+SACK) arrives. (tcp_sacktag_write_queue()) 940 * 2. Retransmission. (tcp_retransmit_skb(), tcp_xmit_retransmit_queue()) 941 * 3. Loss detection event of two flavors: 942 * A. Scoreboard estimator decided the packet is lost. 943 * A'. Reno "three dupacks" marks head of queue lost. 944 * A''. Its FACK modification, head until snd.fack is lost. 945 * B. SACK arrives sacking SND.NXT at the moment, when the 946 * segment was retransmitted. 947 * 4. D-SACK added new rule: D-SACK changes any tag to S. 948 * 949 * It is pleasant to note, that state diagram turns out to be commutative, 950 * so that we are allowed not to be bothered by order of our actions, 951 * when multiple events arrive simultaneously. (see the function below). 952 * 953 * Reordering detection. 954 * -------------------- 955 * Reordering metric is maximal distance, which a packet can be displaced 956 * in packet stream. With SACKs we can estimate it: 957 * 958 * 1. SACK fills old hole and the corresponding segment was not 959 * ever retransmitted -> reordering. Alas, we cannot use it 960 * when segment was retransmitted. 961 * 2. The last flaw is solved with D-SACK. D-SACK arrives 962 * for retransmitted and already SACKed segment -> reordering.. 963 * Both of these heuristics are not used in Loss state, when we cannot 964 * account for retransmits accurately. 965 * 966 * SACK block validation. 967 * ---------------------- 968 * 969 * SACK block range validation checks that the received SACK block fits to 970 * the expected sequence limits, i.e., it is between SND.UNA and SND.NXT. 971 * Note that SND.UNA is not included to the range though being valid because 972 * it means that the receiver is rather inconsistent with itself reporting 973 * SACK reneging when it should advance SND.UNA. Such SACK block this is 974 * perfectly valid, however, in light of RFC2018 which explicitly states 975 * that "SACK block MUST reflect the newest segment. Even if the newest 976 * segment is going to be discarded ...", not that it looks very clever 977 * in case of head skb. Due to potentional receiver driven attacks, we 978 * choose to avoid immediate execution of a walk in write queue due to 979 * reneging and defer head skb's loss recovery to standard loss recovery 980 * procedure that will eventually trigger (nothing forbids us doing this). 981 * 982 * Implements also blockage to start_seq wrap-around. Problem lies in the 983 * fact that though start_seq (s) is before end_seq (i.e., not reversed), 984 * there's no guarantee that it will be before snd_nxt (n). The problem 985 * happens when start_seq resides between end_seq wrap (e_w) and snd_nxt 986 * wrap (s_w): 987 * 988 * <- outs wnd -> <- wrapzone -> 989 * u e n u_w e_w s n_w 990 * | | | | | | | 991 * |<------------+------+----- TCP seqno space --------------+---------->| 992 * ...-- <2^31 ->| |<--------... 993 * ...---- >2^31 ------>| |<--------... 994 * 995 * Current code wouldn't be vulnerable but it's better still to discard such 996 * crazy SACK blocks. Doing this check for start_seq alone closes somewhat 997 * similar case (end_seq after snd_nxt wrap) as earlier reversed check in 998 * snd_nxt wrap -> snd_una region will then become "well defined", i.e., 999 * equal to the ideal case (infinite seqno space without wrap caused issues). 1000 * 1001 * With D-SACK the lower bound is extended to cover sequence space below 1002 * SND.UNA down to undo_marker, which is the last point of interest. Yet 1003 * again, D-SACK block must not to go across snd_una (for the same reason as 1004 * for the normal SACK blocks, explained above). But there all simplicity 1005 * ends, TCP might receive valid D-SACKs below that. As long as they reside 1006 * fully below undo_marker they do not affect behavior in anyway and can 1007 * therefore be safely ignored. In rare cases (which are more or less 1008 * theoretical ones), the D-SACK will nicely cross that boundary due to skb 1009 * fragmentation and packet reordering past skb's retransmission. To consider 1010 * them correctly, the acceptable range must be extended even more though 1011 * the exact amount is rather hard to quantify. However, tp->max_window can 1012 * be used as an exaggerated estimate. 1013 */ 1014 static bool tcp_is_sackblock_valid(struct tcp_sock *tp, bool is_dsack, 1015 u32 start_seq, u32 end_seq) 1016 { 1017 /* Too far in future, or reversed (interpretation is ambiguous) */ 1018 if (after(end_seq, tp->snd_nxt) || !before(start_seq, end_seq)) 1019 return false; 1020 1021 /* Nasty start_seq wrap-around check (see comments above) */ 1022 if (!before(start_seq, tp->snd_nxt)) 1023 return false; 1024 1025 /* In outstanding window? ...This is valid exit for D-SACKs too. 1026 * start_seq == snd_una is non-sensical (see comments above) 1027 */ 1028 if (after(start_seq, tp->snd_una)) 1029 return true; 1030 1031 if (!is_dsack || !tp->undo_marker) 1032 return false; 1033 1034 /* ...Then it's D-SACK, and must reside below snd_una completely */ 1035 if (after(end_seq, tp->snd_una)) 1036 return false; 1037 1038 if (!before(start_seq, tp->undo_marker)) 1039 return true; 1040 1041 /* Too old */ 1042 if (!after(end_seq, tp->undo_marker)) 1043 return false; 1044 1045 /* Undo_marker boundary crossing (overestimates a lot). Known already: 1046 * start_seq < undo_marker and end_seq >= undo_marker. 1047 */ 1048 return !before(start_seq, end_seq - tp->max_window); 1049 } 1050 1051 static bool tcp_check_dsack(struct sock *sk, const struct sk_buff *ack_skb, 1052 struct tcp_sack_block_wire *sp, int num_sacks, 1053 u32 prior_snd_una) 1054 { 1055 struct tcp_sock *tp = tcp_sk(sk); 1056 u32 start_seq_0 = get_unaligned_be32(&sp[0].start_seq); 1057 u32 end_seq_0 = get_unaligned_be32(&sp[0].end_seq); 1058 bool dup_sack = false; 1059 1060 if (before(start_seq_0, TCP_SKB_CB(ack_skb)->ack_seq)) { 1061 dup_sack = true; 1062 tcp_dsack_seen(tp); 1063 NET_INC_STATS_BH(sock_net(sk), LINUX_MIB_TCPDSACKRECV); 1064 } else if (num_sacks > 1) { 1065 u32 end_seq_1 = get_unaligned_be32(&sp[1].end_seq); 1066 u32 start_seq_1 = get_unaligned_be32(&sp[1].start_seq); 1067 1068 if (!after(end_seq_0, end_seq_1) && 1069 !before(start_seq_0, start_seq_1)) { 1070 dup_sack = true; 1071 tcp_dsack_seen(tp); 1072 NET_INC_STATS_BH(sock_net(sk), 1073 LINUX_MIB_TCPDSACKOFORECV); 1074 } 1075 } 1076 1077 /* D-SACK for already forgotten data... Do dumb counting. */ 1078 if (dup_sack && tp->undo_marker && tp->undo_retrans > 0 && 1079 !after(end_seq_0, prior_snd_una) && 1080 after(end_seq_0, tp->undo_marker)) 1081 tp->undo_retrans--; 1082 1083 return dup_sack; 1084 } 1085 1086 struct tcp_sacktag_state { 1087 int reord; 1088 int fack_count; 1089 /* Timestamps for earliest and latest never-retransmitted segment 1090 * that was SACKed. RTO needs the earliest RTT to stay conservative, 1091 * but congestion control should still get an accurate delay signal. 1092 */ 1093 struct skb_mstamp first_sackt; 1094 struct skb_mstamp last_sackt; 1095 int flag; 1096 }; 1097 1098 /* Check if skb is fully within the SACK block. In presence of GSO skbs, 1099 * the incoming SACK may not exactly match but we can find smaller MSS 1100 * aligned portion of it that matches. Therefore we might need to fragment 1101 * which may fail and creates some hassle (caller must handle error case 1102 * returns). 1103 * 1104 * FIXME: this could be merged to shift decision code 1105 */ 1106 static int tcp_match_skb_to_sack(struct sock *sk, struct sk_buff *skb, 1107 u32 start_seq, u32 end_seq) 1108 { 1109 int err; 1110 bool in_sack; 1111 unsigned int pkt_len; 1112 unsigned int mss; 1113 1114 in_sack = !after(start_seq, TCP_SKB_CB(skb)->seq) && 1115 !before(end_seq, TCP_SKB_CB(skb)->end_seq); 1116 1117 if (tcp_skb_pcount(skb) > 1 && !in_sack && 1118 after(TCP_SKB_CB(skb)->end_seq, start_seq)) { 1119 mss = tcp_skb_mss(skb); 1120 in_sack = !after(start_seq, TCP_SKB_CB(skb)->seq); 1121 1122 if (!in_sack) { 1123 pkt_len = start_seq - TCP_SKB_CB(skb)->seq; 1124 if (pkt_len < mss) 1125 pkt_len = mss; 1126 } else { 1127 pkt_len = end_seq - TCP_SKB_CB(skb)->seq; 1128 if (pkt_len < mss) 1129 return -EINVAL; 1130 } 1131 1132 /* Round if necessary so that SACKs cover only full MSSes 1133 * and/or the remaining small portion (if present) 1134 */ 1135 if (pkt_len > mss) { 1136 unsigned int new_len = (pkt_len / mss) * mss; 1137 if (!in_sack && new_len < pkt_len) { 1138 new_len += mss; 1139 if (new_len >= skb->len) 1140 return 0; 1141 } 1142 pkt_len = new_len; 1143 } 1144 err = tcp_fragment(sk, skb, pkt_len, mss, GFP_ATOMIC); 1145 if (err < 0) 1146 return err; 1147 } 1148 1149 return in_sack; 1150 } 1151 1152 /* Mark the given newly-SACKed range as such, adjusting counters and hints. */ 1153 static u8 tcp_sacktag_one(struct sock *sk, 1154 struct tcp_sacktag_state *state, u8 sacked, 1155 u32 start_seq, u32 end_seq, 1156 int dup_sack, int pcount, 1157 const struct skb_mstamp *xmit_time) 1158 { 1159 struct tcp_sock *tp = tcp_sk(sk); 1160 int fack_count = state->fack_count; 1161 1162 /* Account D-SACK for retransmitted packet. */ 1163 if (dup_sack && (sacked & TCPCB_RETRANS)) { 1164 if (tp->undo_marker && tp->undo_retrans > 0 && 1165 after(end_seq, tp->undo_marker)) 1166 tp->undo_retrans--; 1167 if (sacked & TCPCB_SACKED_ACKED) 1168 state->reord = min(fack_count, state->reord); 1169 } 1170 1171 /* Nothing to do; acked frame is about to be dropped (was ACKed). */ 1172 if (!after(end_seq, tp->snd_una)) 1173 return sacked; 1174 1175 if (!(sacked & TCPCB_SACKED_ACKED)) { 1176 tcp_rack_advance(tp, xmit_time, sacked); 1177 1178 if (sacked & TCPCB_SACKED_RETRANS) { 1179 /* If the segment is not tagged as lost, 1180 * we do not clear RETRANS, believing 1181 * that retransmission is still in flight. 1182 */ 1183 if (sacked & TCPCB_LOST) { 1184 sacked &= ~(TCPCB_LOST|TCPCB_SACKED_RETRANS); 1185 tp->lost_out -= pcount; 1186 tp->retrans_out -= pcount; 1187 } 1188 } else { 1189 if (!(sacked & TCPCB_RETRANS)) { 1190 /* New sack for not retransmitted frame, 1191 * which was in hole. It is reordering. 1192 */ 1193 if (before(start_seq, 1194 tcp_highest_sack_seq(tp))) 1195 state->reord = min(fack_count, 1196 state->reord); 1197 if (!after(end_seq, tp->high_seq)) 1198 state->flag |= FLAG_ORIG_SACK_ACKED; 1199 if (state->first_sackt.v64 == 0) 1200 state->first_sackt = *xmit_time; 1201 state->last_sackt = *xmit_time; 1202 } 1203 1204 if (sacked & TCPCB_LOST) { 1205 sacked &= ~TCPCB_LOST; 1206 tp->lost_out -= pcount; 1207 } 1208 } 1209 1210 sacked |= TCPCB_SACKED_ACKED; 1211 state->flag |= FLAG_DATA_SACKED; 1212 tp->sacked_out += pcount; 1213 1214 fack_count += pcount; 1215 1216 /* Lost marker hint past SACKed? Tweak RFC3517 cnt */ 1217 if (!tcp_is_fack(tp) && tp->lost_skb_hint && 1218 before(start_seq, TCP_SKB_CB(tp->lost_skb_hint)->seq)) 1219 tp->lost_cnt_hint += pcount; 1220 1221 if (fack_count > tp->fackets_out) 1222 tp->fackets_out = fack_count; 1223 } 1224 1225 /* D-SACK. We can detect redundant retransmission in S|R and plain R 1226 * frames and clear it. undo_retrans is decreased above, L|R frames 1227 * are accounted above as well. 1228 */ 1229 if (dup_sack && (sacked & TCPCB_SACKED_RETRANS)) { 1230 sacked &= ~TCPCB_SACKED_RETRANS; 1231 tp->retrans_out -= pcount; 1232 } 1233 1234 return sacked; 1235 } 1236 1237 /* Shift newly-SACKed bytes from this skb to the immediately previous 1238 * already-SACKed sk_buff. Mark the newly-SACKed bytes as such. 1239 */ 1240 static bool tcp_shifted_skb(struct sock *sk, struct sk_buff *skb, 1241 struct tcp_sacktag_state *state, 1242 unsigned int pcount, int shifted, int mss, 1243 bool dup_sack) 1244 { 1245 struct tcp_sock *tp = tcp_sk(sk); 1246 struct sk_buff *prev = tcp_write_queue_prev(sk, skb); 1247 u32 start_seq = TCP_SKB_CB(skb)->seq; /* start of newly-SACKed */ 1248 u32 end_seq = start_seq + shifted; /* end of newly-SACKed */ 1249 1250 BUG_ON(!pcount); 1251 1252 /* Adjust counters and hints for the newly sacked sequence 1253 * range but discard the return value since prev is already 1254 * marked. We must tag the range first because the seq 1255 * advancement below implicitly advances 1256 * tcp_highest_sack_seq() when skb is highest_sack. 1257 */ 1258 tcp_sacktag_one(sk, state, TCP_SKB_CB(skb)->sacked, 1259 start_seq, end_seq, dup_sack, pcount, 1260 &skb->skb_mstamp); 1261 1262 if (skb == tp->lost_skb_hint) 1263 tp->lost_cnt_hint += pcount; 1264 1265 TCP_SKB_CB(prev)->end_seq += shifted; 1266 TCP_SKB_CB(skb)->seq += shifted; 1267 1268 tcp_skb_pcount_add(prev, pcount); 1269 BUG_ON(tcp_skb_pcount(skb) < pcount); 1270 tcp_skb_pcount_add(skb, -pcount); 1271 1272 /* When we're adding to gso_segs == 1, gso_size will be zero, 1273 * in theory this shouldn't be necessary but as long as DSACK 1274 * code can come after this skb later on it's better to keep 1275 * setting gso_size to something. 1276 */ 1277 if (!TCP_SKB_CB(prev)->tcp_gso_size) 1278 TCP_SKB_CB(prev)->tcp_gso_size = mss; 1279 1280 /* CHECKME: To clear or not to clear? Mimics normal skb currently */ 1281 if (tcp_skb_pcount(skb) <= 1) 1282 TCP_SKB_CB(skb)->tcp_gso_size = 0; 1283 1284 /* Difference in this won't matter, both ACKed by the same cumul. ACK */ 1285 TCP_SKB_CB(prev)->sacked |= (TCP_SKB_CB(skb)->sacked & TCPCB_EVER_RETRANS); 1286 1287 if (skb->len > 0) { 1288 BUG_ON(!tcp_skb_pcount(skb)); 1289 NET_INC_STATS_BH(sock_net(sk), LINUX_MIB_SACKSHIFTED); 1290 return false; 1291 } 1292 1293 /* Whole SKB was eaten :-) */ 1294 1295 if (skb == tp->retransmit_skb_hint) 1296 tp->retransmit_skb_hint = prev; 1297 if (skb == tp->lost_skb_hint) { 1298 tp->lost_skb_hint = prev; 1299 tp->lost_cnt_hint -= tcp_skb_pcount(prev); 1300 } 1301 1302 TCP_SKB_CB(prev)->tcp_flags |= TCP_SKB_CB(skb)->tcp_flags; 1303 if (TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN) 1304 TCP_SKB_CB(prev)->end_seq++; 1305 1306 if (skb == tcp_highest_sack(sk)) 1307 tcp_advance_highest_sack(sk, skb); 1308 1309 tcp_unlink_write_queue(skb, sk); 1310 sk_wmem_free_skb(sk, skb); 1311 1312 NET_INC_STATS_BH(sock_net(sk), LINUX_MIB_SACKMERGED); 1313 1314 return true; 1315 } 1316 1317 /* I wish gso_size would have a bit more sane initialization than 1318 * something-or-zero which complicates things 1319 */ 1320 static int tcp_skb_seglen(const struct sk_buff *skb) 1321 { 1322 return tcp_skb_pcount(skb) == 1 ? skb->len : tcp_skb_mss(skb); 1323 } 1324 1325 /* Shifting pages past head area doesn't work */ 1326 static int skb_can_shift(const struct sk_buff *skb) 1327 { 1328 return !skb_headlen(skb) && skb_is_nonlinear(skb); 1329 } 1330 1331 /* Try collapsing SACK blocks spanning across multiple skbs to a single 1332 * skb. 1333 */ 1334 static struct sk_buff *tcp_shift_skb_data(struct sock *sk, struct sk_buff *skb, 1335 struct tcp_sacktag_state *state, 1336 u32 start_seq, u32 end_seq, 1337 bool dup_sack) 1338 { 1339 struct tcp_sock *tp = tcp_sk(sk); 1340 struct sk_buff *prev; 1341 int mss; 1342 int pcount = 0; 1343 int len; 1344 int in_sack; 1345 1346 if (!sk_can_gso(sk)) 1347 goto fallback; 1348 1349 /* Normally R but no L won't result in plain S */ 1350 if (!dup_sack && 1351 (TCP_SKB_CB(skb)->sacked & (TCPCB_LOST|TCPCB_SACKED_RETRANS)) == TCPCB_SACKED_RETRANS) 1352 goto fallback; 1353 if (!skb_can_shift(skb)) 1354 goto fallback; 1355 /* This frame is about to be dropped (was ACKed). */ 1356 if (!after(TCP_SKB_CB(skb)->end_seq, tp->snd_una)) 1357 goto fallback; 1358 1359 /* Can only happen with delayed DSACK + discard craziness */ 1360 if (unlikely(skb == tcp_write_queue_head(sk))) 1361 goto fallback; 1362 prev = tcp_write_queue_prev(sk, skb); 1363 1364 if ((TCP_SKB_CB(prev)->sacked & TCPCB_TAGBITS) != TCPCB_SACKED_ACKED) 1365 goto fallback; 1366 1367 in_sack = !after(start_seq, TCP_SKB_CB(skb)->seq) && 1368 !before(end_seq, TCP_SKB_CB(skb)->end_seq); 1369 1370 if (in_sack) { 1371 len = skb->len; 1372 pcount = tcp_skb_pcount(skb); 1373 mss = tcp_skb_seglen(skb); 1374 1375 /* TODO: Fix DSACKs to not fragment already SACKed and we can 1376 * drop this restriction as unnecessary 1377 */ 1378 if (mss != tcp_skb_seglen(prev)) 1379 goto fallback; 1380 } else { 1381 if (!after(TCP_SKB_CB(skb)->end_seq, start_seq)) 1382 goto noop; 1383 /* CHECKME: This is non-MSS split case only?, this will 1384 * cause skipped skbs due to advancing loop btw, original 1385 * has that feature too 1386 */ 1387 if (tcp_skb_pcount(skb) <= 1) 1388 goto noop; 1389 1390 in_sack = !after(start_seq, TCP_SKB_CB(skb)->seq); 1391 if (!in_sack) { 1392 /* TODO: head merge to next could be attempted here 1393 * if (!after(TCP_SKB_CB(skb)->end_seq, end_seq)), 1394 * though it might not be worth of the additional hassle 1395 * 1396 * ...we can probably just fallback to what was done 1397 * previously. We could try merging non-SACKed ones 1398 * as well but it probably isn't going to buy off 1399 * because later SACKs might again split them, and 1400 * it would make skb timestamp tracking considerably 1401 * harder problem. 1402 */ 1403 goto fallback; 1404 } 1405 1406 len = end_seq - TCP_SKB_CB(skb)->seq; 1407 BUG_ON(len < 0); 1408 BUG_ON(len > skb->len); 1409 1410 /* MSS boundaries should be honoured or else pcount will 1411 * severely break even though it makes things bit trickier. 1412 * Optimize common case to avoid most of the divides 1413 */ 1414 mss = tcp_skb_mss(skb); 1415 1416 /* TODO: Fix DSACKs to not fragment already SACKed and we can 1417 * drop this restriction as unnecessary 1418 */ 1419 if (mss != tcp_skb_seglen(prev)) 1420 goto fallback; 1421 1422 if (len == mss) { 1423 pcount = 1; 1424 } else if (len < mss) { 1425 goto noop; 1426 } else { 1427 pcount = len / mss; 1428 len = pcount * mss; 1429 } 1430 } 1431 1432 /* tcp_sacktag_one() won't SACK-tag ranges below snd_una */ 1433 if (!after(TCP_SKB_CB(skb)->seq + len, tp->snd_una)) 1434 goto fallback; 1435 1436 if (!skb_shift(prev, skb, len)) 1437 goto fallback; 1438 if (!tcp_shifted_skb(sk, skb, state, pcount, len, mss, dup_sack)) 1439 goto out; 1440 1441 /* Hole filled allows collapsing with the next as well, this is very 1442 * useful when hole on every nth skb pattern happens 1443 */ 1444 if (prev == tcp_write_queue_tail(sk)) 1445 goto out; 1446 skb = tcp_write_queue_next(sk, prev); 1447 1448 if (!skb_can_shift(skb) || 1449 (skb == tcp_send_head(sk)) || 1450 ((TCP_SKB_CB(skb)->sacked & TCPCB_TAGBITS) != TCPCB_SACKED_ACKED) || 1451 (mss != tcp_skb_seglen(skb))) 1452 goto out; 1453 1454 len = skb->len; 1455 if (skb_shift(prev, skb, len)) { 1456 pcount += tcp_skb_pcount(skb); 1457 tcp_shifted_skb(sk, skb, state, tcp_skb_pcount(skb), len, mss, 0); 1458 } 1459 1460 out: 1461 state->fack_count += pcount; 1462 return prev; 1463 1464 noop: 1465 return skb; 1466 1467 fallback: 1468 NET_INC_STATS_BH(sock_net(sk), LINUX_MIB_SACKSHIFTFALLBACK); 1469 return NULL; 1470 } 1471 1472 static struct sk_buff *tcp_sacktag_walk(struct sk_buff *skb, struct sock *sk, 1473 struct tcp_sack_block *next_dup, 1474 struct tcp_sacktag_state *state, 1475 u32 start_seq, u32 end_seq, 1476 bool dup_sack_in) 1477 { 1478 struct tcp_sock *tp = tcp_sk(sk); 1479 struct sk_buff *tmp; 1480 1481 tcp_for_write_queue_from(skb, sk) { 1482 int in_sack = 0; 1483 bool dup_sack = dup_sack_in; 1484 1485 if (skb == tcp_send_head(sk)) 1486 break; 1487 1488 /* queue is in-order => we can short-circuit the walk early */ 1489 if (!before(TCP_SKB_CB(skb)->seq, end_seq)) 1490 break; 1491 1492 if (next_dup && 1493 before(TCP_SKB_CB(skb)->seq, next_dup->end_seq)) { 1494 in_sack = tcp_match_skb_to_sack(sk, skb, 1495 next_dup->start_seq, 1496 next_dup->end_seq); 1497 if (in_sack > 0) 1498 dup_sack = true; 1499 } 1500 1501 /* skb reference here is a bit tricky to get right, since 1502 * shifting can eat and free both this skb and the next, 1503 * so not even _safe variant of the loop is enough. 1504 */ 1505 if (in_sack <= 0) { 1506 tmp = tcp_shift_skb_data(sk, skb, state, 1507 start_seq, end_seq, dup_sack); 1508 if (tmp) { 1509 if (tmp != skb) { 1510 skb = tmp; 1511 continue; 1512 } 1513 1514 in_sack = 0; 1515 } else { 1516 in_sack = tcp_match_skb_to_sack(sk, skb, 1517 start_seq, 1518 end_seq); 1519 } 1520 } 1521 1522 if (unlikely(in_sack < 0)) 1523 break; 1524 1525 if (in_sack) { 1526 TCP_SKB_CB(skb)->sacked = 1527 tcp_sacktag_one(sk, 1528 state, 1529 TCP_SKB_CB(skb)->sacked, 1530 TCP_SKB_CB(skb)->seq, 1531 TCP_SKB_CB(skb)->end_seq, 1532 dup_sack, 1533 tcp_skb_pcount(skb), 1534 &skb->skb_mstamp); 1535 1536 if (!before(TCP_SKB_CB(skb)->seq, 1537 tcp_highest_sack_seq(tp))) 1538 tcp_advance_highest_sack(sk, skb); 1539 } 1540 1541 state->fack_count += tcp_skb_pcount(skb); 1542 } 1543 return skb; 1544 } 1545 1546 /* Avoid all extra work that is being done by sacktag while walking in 1547 * a normal way 1548 */ 1549 static struct sk_buff *tcp_sacktag_skip(struct sk_buff *skb, struct sock *sk, 1550 struct tcp_sacktag_state *state, 1551 u32 skip_to_seq) 1552 { 1553 tcp_for_write_queue_from(skb, sk) { 1554 if (skb == tcp_send_head(sk)) 1555 break; 1556 1557 if (after(TCP_SKB_CB(skb)->end_seq, skip_to_seq)) 1558 break; 1559 1560 state->fack_count += tcp_skb_pcount(skb); 1561 } 1562 return skb; 1563 } 1564 1565 static struct sk_buff *tcp_maybe_skipping_dsack(struct sk_buff *skb, 1566 struct sock *sk, 1567 struct tcp_sack_block *next_dup, 1568 struct tcp_sacktag_state *state, 1569 u32 skip_to_seq) 1570 { 1571 if (!next_dup) 1572 return skb; 1573 1574 if (before(next_dup->start_seq, skip_to_seq)) { 1575 skb = tcp_sacktag_skip(skb, sk, state, next_dup->start_seq); 1576 skb = tcp_sacktag_walk(skb, sk, NULL, state, 1577 next_dup->start_seq, next_dup->end_seq, 1578 1); 1579 } 1580 1581 return skb; 1582 } 1583 1584 static int tcp_sack_cache_ok(const struct tcp_sock *tp, const struct tcp_sack_block *cache) 1585 { 1586 return cache < tp->recv_sack_cache + ARRAY_SIZE(tp->recv_sack_cache); 1587 } 1588 1589 static int 1590 tcp_sacktag_write_queue(struct sock *sk, const struct sk_buff *ack_skb, 1591 u32 prior_snd_una, struct tcp_sacktag_state *state) 1592 { 1593 struct tcp_sock *tp = tcp_sk(sk); 1594 const unsigned char *ptr = (skb_transport_header(ack_skb) + 1595 TCP_SKB_CB(ack_skb)->sacked); 1596 struct tcp_sack_block_wire *sp_wire = (struct tcp_sack_block_wire *)(ptr+2); 1597 struct tcp_sack_block sp[TCP_NUM_SACKS]; 1598 struct tcp_sack_block *cache; 1599 struct sk_buff *skb; 1600 int num_sacks = min(TCP_NUM_SACKS, (ptr[1] - TCPOLEN_SACK_BASE) >> 3); 1601 int used_sacks; 1602 bool found_dup_sack = false; 1603 int i, j; 1604 int first_sack_index; 1605 1606 state->flag = 0; 1607 state->reord = tp->packets_out; 1608 1609 if (!tp->sacked_out) { 1610 if (WARN_ON(tp->fackets_out)) 1611 tp->fackets_out = 0; 1612 tcp_highest_sack_reset(sk); 1613 } 1614 1615 found_dup_sack = tcp_check_dsack(sk, ack_skb, sp_wire, 1616 num_sacks, prior_snd_una); 1617 if (found_dup_sack) 1618 state->flag |= FLAG_DSACKING_ACK; 1619 1620 /* Eliminate too old ACKs, but take into 1621 * account more or less fresh ones, they can 1622 * contain valid SACK info. 1623 */ 1624 if (before(TCP_SKB_CB(ack_skb)->ack_seq, prior_snd_una - tp->max_window)) 1625 return 0; 1626 1627 if (!tp->packets_out) 1628 goto out; 1629 1630 used_sacks = 0; 1631 first_sack_index = 0; 1632 for (i = 0; i < num_sacks; i++) { 1633 bool dup_sack = !i && found_dup_sack; 1634 1635 sp[used_sacks].start_seq = get_unaligned_be32(&sp_wire[i].start_seq); 1636 sp[used_sacks].end_seq = get_unaligned_be32(&sp_wire[i].end_seq); 1637 1638 if (!tcp_is_sackblock_valid(tp, dup_sack, 1639 sp[used_sacks].start_seq, 1640 sp[used_sacks].end_seq)) { 1641 int mib_idx; 1642 1643 if (dup_sack) { 1644 if (!tp->undo_marker) 1645 mib_idx = LINUX_MIB_TCPDSACKIGNOREDNOUNDO; 1646 else 1647 mib_idx = LINUX_MIB_TCPDSACKIGNOREDOLD; 1648 } else { 1649 /* Don't count olds caused by ACK reordering */ 1650 if ((TCP_SKB_CB(ack_skb)->ack_seq != tp->snd_una) && 1651 !after(sp[used_sacks].end_seq, tp->snd_una)) 1652 continue; 1653 mib_idx = LINUX_MIB_TCPSACKDISCARD; 1654 } 1655 1656 NET_INC_STATS_BH(sock_net(sk), mib_idx); 1657 if (i == 0) 1658 first_sack_index = -1; 1659 continue; 1660 } 1661 1662 /* Ignore very old stuff early */ 1663 if (!after(sp[used_sacks].end_seq, prior_snd_una)) 1664 continue; 1665 1666 used_sacks++; 1667 } 1668 1669 /* order SACK blocks to allow in order walk of the retrans queue */ 1670 for (i = used_sacks - 1; i > 0; i--) { 1671 for (j = 0; j < i; j++) { 1672 if (after(sp[j].start_seq, sp[j + 1].start_seq)) { 1673 swap(sp[j], sp[j + 1]); 1674 1675 /* Track where the first SACK block goes to */ 1676 if (j == first_sack_index) 1677 first_sack_index = j + 1; 1678 } 1679 } 1680 } 1681 1682 skb = tcp_write_queue_head(sk); 1683 state->fack_count = 0; 1684 i = 0; 1685 1686 if (!tp->sacked_out) { 1687 /* It's already past, so skip checking against it */ 1688 cache = tp->recv_sack_cache + ARRAY_SIZE(tp->recv_sack_cache); 1689 } else { 1690 cache = tp->recv_sack_cache; 1691 /* Skip empty blocks in at head of the cache */ 1692 while (tcp_sack_cache_ok(tp, cache) && !cache->start_seq && 1693 !cache->end_seq) 1694 cache++; 1695 } 1696 1697 while (i < used_sacks) { 1698 u32 start_seq = sp[i].start_seq; 1699 u32 end_seq = sp[i].end_seq; 1700 bool dup_sack = (found_dup_sack && (i == first_sack_index)); 1701 struct tcp_sack_block *next_dup = NULL; 1702 1703 if (found_dup_sack && ((i + 1) == first_sack_index)) 1704 next_dup = &sp[i + 1]; 1705 1706 /* Skip too early cached blocks */ 1707 while (tcp_sack_cache_ok(tp, cache) && 1708 !before(start_seq, cache->end_seq)) 1709 cache++; 1710 1711 /* Can skip some work by looking recv_sack_cache? */ 1712 if (tcp_sack_cache_ok(tp, cache) && !dup_sack && 1713 after(end_seq, cache->start_seq)) { 1714 1715 /* Head todo? */ 1716 if (before(start_seq, cache->start_seq)) { 1717 skb = tcp_sacktag_skip(skb, sk, state, 1718 start_seq); 1719 skb = tcp_sacktag_walk(skb, sk, next_dup, 1720 state, 1721 start_seq, 1722 cache->start_seq, 1723 dup_sack); 1724 } 1725 1726 /* Rest of the block already fully processed? */ 1727 if (!after(end_seq, cache->end_seq)) 1728 goto advance_sp; 1729 1730 skb = tcp_maybe_skipping_dsack(skb, sk, next_dup, 1731 state, 1732 cache->end_seq); 1733 1734 /* ...tail remains todo... */ 1735 if (tcp_highest_sack_seq(tp) == cache->end_seq) { 1736 /* ...but better entrypoint exists! */ 1737 skb = tcp_highest_sack(sk); 1738 if (!skb) 1739 break; 1740 state->fack_count = tp->fackets_out; 1741 cache++; 1742 goto walk; 1743 } 1744 1745 skb = tcp_sacktag_skip(skb, sk, state, cache->end_seq); 1746 /* Check overlap against next cached too (past this one already) */ 1747 cache++; 1748 continue; 1749 } 1750 1751 if (!before(start_seq, tcp_highest_sack_seq(tp))) { 1752 skb = tcp_highest_sack(sk); 1753 if (!skb) 1754 break; 1755 state->fack_count = tp->fackets_out; 1756 } 1757 skb = tcp_sacktag_skip(skb, sk, state, start_seq); 1758 1759 walk: 1760 skb = tcp_sacktag_walk(skb, sk, next_dup, state, 1761 start_seq, end_seq, dup_sack); 1762 1763 advance_sp: 1764 i++; 1765 } 1766 1767 /* Clear the head of the cache sack blocks so we can skip it next time */ 1768 for (i = 0; i < ARRAY_SIZE(tp->recv_sack_cache) - used_sacks; i++) { 1769 tp->recv_sack_cache[i].start_seq = 0; 1770 tp->recv_sack_cache[i].end_seq = 0; 1771 } 1772 for (j = 0; j < used_sacks; j++) 1773 tp->recv_sack_cache[i++] = sp[j]; 1774 1775 if ((state->reord < tp->fackets_out) && 1776 ((inet_csk(sk)->icsk_ca_state != TCP_CA_Loss) || tp->undo_marker)) 1777 tcp_update_reordering(sk, tp->fackets_out - state->reord, 0); 1778 1779 tcp_verify_left_out(tp); 1780 out: 1781 1782 #if FASTRETRANS_DEBUG > 0 1783 WARN_ON((int)tp->sacked_out < 0); 1784 WARN_ON((int)tp->lost_out < 0); 1785 WARN_ON((int)tp->retrans_out < 0); 1786 WARN_ON((int)tcp_packets_in_flight(tp) < 0); 1787 #endif 1788 return state->flag; 1789 } 1790 1791 /* Limits sacked_out so that sum with lost_out isn't ever larger than 1792 * packets_out. Returns false if sacked_out adjustement wasn't necessary. 1793 */ 1794 static bool tcp_limit_reno_sacked(struct tcp_sock *tp) 1795 { 1796 u32 holes; 1797 1798 holes = max(tp->lost_out, 1U); 1799 holes = min(holes, tp->packets_out); 1800 1801 if ((tp->sacked_out + holes) > tp->packets_out) { 1802 tp->sacked_out = tp->packets_out - holes; 1803 return true; 1804 } 1805 return false; 1806 } 1807 1808 /* If we receive more dupacks than we expected counting segments 1809 * in assumption of absent reordering, interpret this as reordering. 1810 * The only another reason could be bug in receiver TCP. 1811 */ 1812 static void tcp_check_reno_reordering(struct sock *sk, const int addend) 1813 { 1814 struct tcp_sock *tp = tcp_sk(sk); 1815 if (tcp_limit_reno_sacked(tp)) 1816 tcp_update_reordering(sk, tp->packets_out + addend, 0); 1817 } 1818 1819 /* Emulate SACKs for SACKless connection: account for a new dupack. */ 1820 1821 static void tcp_add_reno_sack(struct sock *sk) 1822 { 1823 struct tcp_sock *tp = tcp_sk(sk); 1824 tp->sacked_out++; 1825 tcp_check_reno_reordering(sk, 0); 1826 tcp_verify_left_out(tp); 1827 } 1828 1829 /* Account for ACK, ACKing some data in Reno Recovery phase. */ 1830 1831 static void tcp_remove_reno_sacks(struct sock *sk, int acked) 1832 { 1833 struct tcp_sock *tp = tcp_sk(sk); 1834 1835 if (acked > 0) { 1836 /* One ACK acked hole. The rest eat duplicate ACKs. */ 1837 if (acked - 1 >= tp->sacked_out) 1838 tp->sacked_out = 0; 1839 else 1840 tp->sacked_out -= acked - 1; 1841 } 1842 tcp_check_reno_reordering(sk, acked); 1843 tcp_verify_left_out(tp); 1844 } 1845 1846 static inline void tcp_reset_reno_sack(struct tcp_sock *tp) 1847 { 1848 tp->sacked_out = 0; 1849 } 1850 1851 void tcp_clear_retrans(struct tcp_sock *tp) 1852 { 1853 tp->retrans_out = 0; 1854 tp->lost_out = 0; 1855 tp->undo_marker = 0; 1856 tp->undo_retrans = -1; 1857 tp->fackets_out = 0; 1858 tp->sacked_out = 0; 1859 } 1860 1861 static inline void tcp_init_undo(struct tcp_sock *tp) 1862 { 1863 tp->undo_marker = tp->snd_una; 1864 /* Retransmission still in flight may cause DSACKs later. */ 1865 tp->undo_retrans = tp->retrans_out ? : -1; 1866 } 1867 1868 /* Enter Loss state. If we detect SACK reneging, forget all SACK information 1869 * and reset tags completely, otherwise preserve SACKs. If receiver 1870 * dropped its ofo queue, we will know this due to reneging detection. 1871 */ 1872 void tcp_enter_loss(struct sock *sk) 1873 { 1874 const struct inet_connection_sock *icsk = inet_csk(sk); 1875 struct tcp_sock *tp = tcp_sk(sk); 1876 struct sk_buff *skb; 1877 bool new_recovery = icsk->icsk_ca_state < TCP_CA_Recovery; 1878 bool is_reneg; /* is receiver reneging on SACKs? */ 1879 1880 /* Reduce ssthresh if it has not yet been made inside this window. */ 1881 if (icsk->icsk_ca_state <= TCP_CA_Disorder || 1882 !after(tp->high_seq, tp->snd_una) || 1883 (icsk->icsk_ca_state == TCP_CA_Loss && !icsk->icsk_retransmits)) { 1884 tp->prior_ssthresh = tcp_current_ssthresh(sk); 1885 tp->snd_ssthresh = icsk->icsk_ca_ops->ssthresh(sk); 1886 tcp_ca_event(sk, CA_EVENT_LOSS); 1887 tcp_init_undo(tp); 1888 } 1889 tp->snd_cwnd = 1; 1890 tp->snd_cwnd_cnt = 0; 1891 tp->snd_cwnd_stamp = tcp_time_stamp; 1892 1893 tp->retrans_out = 0; 1894 tp->lost_out = 0; 1895 1896 if (tcp_is_reno(tp)) 1897 tcp_reset_reno_sack(tp); 1898 1899 skb = tcp_write_queue_head(sk); 1900 is_reneg = skb && (TCP_SKB_CB(skb)->sacked & TCPCB_SACKED_ACKED); 1901 if (is_reneg) { 1902 NET_INC_STATS_BH(sock_net(sk), LINUX_MIB_TCPSACKRENEGING); 1903 tp->sacked_out = 0; 1904 tp->fackets_out = 0; 1905 } 1906 tcp_clear_all_retrans_hints(tp); 1907 1908 tcp_for_write_queue(skb, sk) { 1909 if (skb == tcp_send_head(sk)) 1910 break; 1911 1912 TCP_SKB_CB(skb)->sacked &= (~TCPCB_TAGBITS)|TCPCB_SACKED_ACKED; 1913 if (!(TCP_SKB_CB(skb)->sacked&TCPCB_SACKED_ACKED) || is_reneg) { 1914 TCP_SKB_CB(skb)->sacked &= ~TCPCB_SACKED_ACKED; 1915 TCP_SKB_CB(skb)->sacked |= TCPCB_LOST; 1916 tp->lost_out += tcp_skb_pcount(skb); 1917 tp->retransmit_high = TCP_SKB_CB(skb)->end_seq; 1918 } 1919 } 1920 tcp_verify_left_out(tp); 1921 1922 /* Timeout in disordered state after receiving substantial DUPACKs 1923 * suggests that the degree of reordering is over-estimated. 1924 */ 1925 if (icsk->icsk_ca_state <= TCP_CA_Disorder && 1926 tp->sacked_out >= sysctl_tcp_reordering) 1927 tp->reordering = min_t(unsigned int, tp->reordering, 1928 sysctl_tcp_reordering); 1929 tcp_set_ca_state(sk, TCP_CA_Loss); 1930 tp->high_seq = tp->snd_nxt; 1931 tcp_ecn_queue_cwr(tp); 1932 1933 /* F-RTO RFC5682 sec 3.1 step 1: retransmit SND.UNA if no previous 1934 * loss recovery is underway except recurring timeout(s) on 1935 * the same SND.UNA (sec 3.2). Disable F-RTO on path MTU probing 1936 */ 1937 tp->frto = sysctl_tcp_frto && 1938 (new_recovery || icsk->icsk_retransmits) && 1939 !inet_csk(sk)->icsk_mtup.probe_size; 1940 } 1941 1942 /* If ACK arrived pointing to a remembered SACK, it means that our 1943 * remembered SACKs do not reflect real state of receiver i.e. 1944 * receiver _host_ is heavily congested (or buggy). 1945 * 1946 * To avoid big spurious retransmission bursts due to transient SACK 1947 * scoreboard oddities that look like reneging, we give the receiver a 1948 * little time (max(RTT/2, 10ms)) to send us some more ACKs that will 1949 * restore sanity to the SACK scoreboard. If the apparent reneging 1950 * persists until this RTO then we'll clear the SACK scoreboard. 1951 */ 1952 static bool tcp_check_sack_reneging(struct sock *sk, int flag) 1953 { 1954 if (flag & FLAG_SACK_RENEGING) { 1955 struct tcp_sock *tp = tcp_sk(sk); 1956 unsigned long delay = max(usecs_to_jiffies(tp->srtt_us >> 4), 1957 msecs_to_jiffies(10)); 1958 1959 inet_csk_reset_xmit_timer(sk, ICSK_TIME_RETRANS, 1960 delay, TCP_RTO_MAX); 1961 return true; 1962 } 1963 return false; 1964 } 1965 1966 static inline int tcp_fackets_out(const struct tcp_sock *tp) 1967 { 1968 return tcp_is_reno(tp) ? tp->sacked_out + 1 : tp->fackets_out; 1969 } 1970 1971 /* Heurestics to calculate number of duplicate ACKs. There's no dupACKs 1972 * counter when SACK is enabled (without SACK, sacked_out is used for 1973 * that purpose). 1974 * 1975 * Instead, with FACK TCP uses fackets_out that includes both SACKed 1976 * segments up to the highest received SACK block so far and holes in 1977 * between them. 1978 * 1979 * With reordering, holes may still be in flight, so RFC3517 recovery 1980 * uses pure sacked_out (total number of SACKed segments) even though 1981 * it violates the RFC that uses duplicate ACKs, often these are equal 1982 * but when e.g. out-of-window ACKs or packet duplication occurs, 1983 * they differ. Since neither occurs due to loss, TCP should really 1984 * ignore them. 1985 */ 1986 static inline int tcp_dupack_heuristics(const struct tcp_sock *tp) 1987 { 1988 return tcp_is_fack(tp) ? tp->fackets_out : tp->sacked_out + 1; 1989 } 1990 1991 static bool tcp_pause_early_retransmit(struct sock *sk, int flag) 1992 { 1993 struct tcp_sock *tp = tcp_sk(sk); 1994 unsigned long delay; 1995 1996 /* Delay early retransmit and entering fast recovery for 1997 * max(RTT/4, 2msec) unless ack has ECE mark, no RTT samples 1998 * available, or RTO is scheduled to fire first. 1999 */ 2000 if (sysctl_tcp_early_retrans < 2 || sysctl_tcp_early_retrans > 3 || 2001 (flag & FLAG_ECE) || !tp->srtt_us) 2002 return false; 2003 2004 delay = max(usecs_to_jiffies(tp->srtt_us >> 5), 2005 msecs_to_jiffies(2)); 2006 2007 if (!time_after(inet_csk(sk)->icsk_timeout, (jiffies + delay))) 2008 return false; 2009 2010 inet_csk_reset_xmit_timer(sk, ICSK_TIME_EARLY_RETRANS, delay, 2011 TCP_RTO_MAX); 2012 return true; 2013 } 2014 2015 /* Linux NewReno/SACK/FACK/ECN state machine. 2016 * -------------------------------------- 2017 * 2018 * "Open" Normal state, no dubious events, fast path. 2019 * "Disorder" In all the respects it is "Open", 2020 * but requires a bit more attention. It is entered when 2021 * we see some SACKs or dupacks. It is split of "Open" 2022 * mainly to move some processing from fast path to slow one. 2023 * "CWR" CWND was reduced due to some Congestion Notification event. 2024 * It can be ECN, ICMP source quench, local device congestion. 2025 * "Recovery" CWND was reduced, we are fast-retransmitting. 2026 * "Loss" CWND was reduced due to RTO timeout or SACK reneging. 2027 * 2028 * tcp_fastretrans_alert() is entered: 2029 * - each incoming ACK, if state is not "Open" 2030 * - when arrived ACK is unusual, namely: 2031 * * SACK 2032 * * Duplicate ACK. 2033 * * ECN ECE. 2034 * 2035 * Counting packets in flight is pretty simple. 2036 * 2037 * in_flight = packets_out - left_out + retrans_out 2038 * 2039 * packets_out is SND.NXT-SND.UNA counted in packets. 2040 * 2041 * retrans_out is number of retransmitted segments. 2042 * 2043 * left_out is number of segments left network, but not ACKed yet. 2044 * 2045 * left_out = sacked_out + lost_out 2046 * 2047 * sacked_out: Packets, which arrived to receiver out of order 2048 * and hence not ACKed. With SACKs this number is simply 2049 * amount of SACKed data. Even without SACKs 2050 * it is easy to give pretty reliable estimate of this number, 2051 * counting duplicate ACKs. 2052 * 2053 * lost_out: Packets lost by network. TCP has no explicit 2054 * "loss notification" feedback from network (for now). 2055 * It means that this number can be only _guessed_. 2056 * Actually, it is the heuristics to predict lossage that 2057 * distinguishes different algorithms. 2058 * 2059 * F.e. after RTO, when all the queue is considered as lost, 2060 * lost_out = packets_out and in_flight = retrans_out. 2061 * 2062 * Essentially, we have now two algorithms counting 2063 * lost packets. 2064 * 2065 * FACK: It is the simplest heuristics. As soon as we decided 2066 * that something is lost, we decide that _all_ not SACKed 2067 * packets until the most forward SACK are lost. I.e. 2068 * lost_out = fackets_out - sacked_out and left_out = fackets_out. 2069 * It is absolutely correct estimate, if network does not reorder 2070 * packets. And it loses any connection to reality when reordering 2071 * takes place. We use FACK by default until reordering 2072 * is suspected on the path to this destination. 2073 * 2074 * NewReno: when Recovery is entered, we assume that one segment 2075 * is lost (classic Reno). While we are in Recovery and 2076 * a partial ACK arrives, we assume that one more packet 2077 * is lost (NewReno). This heuristics are the same in NewReno 2078 * and SACK. 2079 * 2080 * Imagine, that's all! Forget about all this shamanism about CWND inflation 2081 * deflation etc. CWND is real congestion window, never inflated, changes 2082 * only according to classic VJ rules. 2083 * 2084 * Really tricky (and requiring careful tuning) part of algorithm 2085 * is hidden in functions tcp_time_to_recover() and tcp_xmit_retransmit_queue(). 2086 * The first determines the moment _when_ we should reduce CWND and, 2087 * hence, slow down forward transmission. In fact, it determines the moment 2088 * when we decide that hole is caused by loss, rather than by a reorder. 2089 * 2090 * tcp_xmit_retransmit_queue() decides, _what_ we should retransmit to fill 2091 * holes, caused by lost packets. 2092 * 2093 * And the most logically complicated part of algorithm is undo 2094 * heuristics. We detect false retransmits due to both too early 2095 * fast retransmit (reordering) and underestimated RTO, analyzing 2096 * timestamps and D-SACKs. When we detect that some segments were 2097 * retransmitted by mistake and CWND reduction was wrong, we undo 2098 * window reduction and abort recovery phase. This logic is hidden 2099 * inside several functions named tcp_try_undo_<something>. 2100 */ 2101 2102 /* This function decides, when we should leave Disordered state 2103 * and enter Recovery phase, reducing congestion window. 2104 * 2105 * Main question: may we further continue forward transmission 2106 * with the same cwnd? 2107 */ 2108 static bool tcp_time_to_recover(struct sock *sk, int flag) 2109 { 2110 struct tcp_sock *tp = tcp_sk(sk); 2111 __u32 packets_out; 2112 2113 /* Trick#1: The loss is proven. */ 2114 if (tp->lost_out) 2115 return true; 2116 2117 /* Not-A-Trick#2 : Classic rule... */ 2118 if (tcp_dupack_heuristics(tp) > tp->reordering) 2119 return true; 2120 2121 /* Trick#4: It is still not OK... But will it be useful to delay 2122 * recovery more? 2123 */ 2124 packets_out = tp->packets_out; 2125 if (packets_out <= tp->reordering && 2126 tp->sacked_out >= max_t(__u32, packets_out/2, sysctl_tcp_reordering) && 2127 !tcp_may_send_now(sk)) { 2128 /* We have nothing to send. This connection is limited 2129 * either by receiver window or by application. 2130 */ 2131 return true; 2132 } 2133 2134 /* If a thin stream is detected, retransmit after first 2135 * received dupack. Employ only if SACK is supported in order 2136 * to avoid possible corner-case series of spurious retransmissions 2137 * Use only if there are no unsent data. 2138 */ 2139 if ((tp->thin_dupack || sysctl_tcp_thin_dupack) && 2140 tcp_stream_is_thin(tp) && tcp_dupack_heuristics(tp) > 1 && 2141 tcp_is_sack(tp) && !tcp_send_head(sk)) 2142 return true; 2143 2144 /* Trick#6: TCP early retransmit, per RFC5827. To avoid spurious 2145 * retransmissions due to small network reorderings, we implement 2146 * Mitigation A.3 in the RFC and delay the retransmission for a short 2147 * interval if appropriate. 2148 */ 2149 if (tp->do_early_retrans && !tp->retrans_out && tp->sacked_out && 2150 (tp->packets_out >= (tp->sacked_out + 1) && tp->packets_out < 4) && 2151 !tcp_may_send_now(sk)) 2152 return !tcp_pause_early_retransmit(sk, flag); 2153 2154 return false; 2155 } 2156 2157 /* Detect loss in event "A" above by marking head of queue up as lost. 2158 * For FACK or non-SACK(Reno) senders, the first "packets" number of segments 2159 * are considered lost. For RFC3517 SACK, a segment is considered lost if it 2160 * has at least tp->reordering SACKed seqments above it; "packets" refers to 2161 * the maximum SACKed segments to pass before reaching this limit. 2162 */ 2163 static void tcp_mark_head_lost(struct sock *sk, int packets, int mark_head) 2164 { 2165 struct tcp_sock *tp = tcp_sk(sk); 2166 struct sk_buff *skb; 2167 int cnt, oldcnt, lost; 2168 unsigned int mss; 2169 /* Use SACK to deduce losses of new sequences sent during recovery */ 2170 const u32 loss_high = tcp_is_sack(tp) ? tp->snd_nxt : tp->high_seq; 2171 2172 WARN_ON(packets > tp->packets_out); 2173 if (tp->lost_skb_hint) { 2174 skb = tp->lost_skb_hint; 2175 cnt = tp->lost_cnt_hint; 2176 /* Head already handled? */ 2177 if (mark_head && skb != tcp_write_queue_head(sk)) 2178 return; 2179 } else { 2180 skb = tcp_write_queue_head(sk); 2181 cnt = 0; 2182 } 2183 2184 tcp_for_write_queue_from(skb, sk) { 2185 if (skb == tcp_send_head(sk)) 2186 break; 2187 /* TODO: do this better */ 2188 /* this is not the most efficient way to do this... */ 2189 tp->lost_skb_hint = skb; 2190 tp->lost_cnt_hint = cnt; 2191 2192 if (after(TCP_SKB_CB(skb)->end_seq, loss_high)) 2193 break; 2194 2195 oldcnt = cnt; 2196 if (tcp_is_fack(tp) || tcp_is_reno(tp) || 2197 (TCP_SKB_CB(skb)->sacked & TCPCB_SACKED_ACKED)) 2198 cnt += tcp_skb_pcount(skb); 2199 2200 if (cnt > packets) { 2201 if ((tcp_is_sack(tp) && !tcp_is_fack(tp)) || 2202 (TCP_SKB_CB(skb)->sacked & TCPCB_SACKED_ACKED) || 2203 (oldcnt >= packets)) 2204 break; 2205 2206 mss = tcp_skb_mss(skb); 2207 /* If needed, chop off the prefix to mark as lost. */ 2208 lost = (packets - oldcnt) * mss; 2209 if (lost < skb->len && 2210 tcp_fragment(sk, skb, lost, mss, GFP_ATOMIC) < 0) 2211 break; 2212 cnt = packets; 2213 } 2214 2215 tcp_skb_mark_lost(tp, skb); 2216 2217 if (mark_head) 2218 break; 2219 } 2220 tcp_verify_left_out(tp); 2221 } 2222 2223 /* Account newly detected lost packet(s) */ 2224 2225 static void tcp_update_scoreboard(struct sock *sk, int fast_rexmit) 2226 { 2227 struct tcp_sock *tp = tcp_sk(sk); 2228 2229 if (tcp_is_reno(tp)) { 2230 tcp_mark_head_lost(sk, 1, 1); 2231 } else if (tcp_is_fack(tp)) { 2232 int lost = tp->fackets_out - tp->reordering; 2233 if (lost <= 0) 2234 lost = 1; 2235 tcp_mark_head_lost(sk, lost, 0); 2236 } else { 2237 int sacked_upto = tp->sacked_out - tp->reordering; 2238 if (sacked_upto >= 0) 2239 tcp_mark_head_lost(sk, sacked_upto, 0); 2240 else if (fast_rexmit) 2241 tcp_mark_head_lost(sk, 1, 1); 2242 } 2243 } 2244 2245 /* CWND moderation, preventing bursts due to too big ACKs 2246 * in dubious situations. 2247 */ 2248 static inline void tcp_moderate_cwnd(struct tcp_sock *tp) 2249 { 2250 tp->snd_cwnd = min(tp->snd_cwnd, 2251 tcp_packets_in_flight(tp) + tcp_max_burst(tp)); 2252 tp->snd_cwnd_stamp = tcp_time_stamp; 2253 } 2254 2255 static bool tcp_tsopt_ecr_before(const struct tcp_sock *tp, u32 when) 2256 { 2257 return tp->rx_opt.saw_tstamp && tp->rx_opt.rcv_tsecr && 2258 before(tp->rx_opt.rcv_tsecr, when); 2259 } 2260 2261 /* skb is spurious retransmitted if the returned timestamp echo 2262 * reply is prior to the skb transmission time 2263 */ 2264 static bool tcp_skb_spurious_retrans(const struct tcp_sock *tp, 2265 const struct sk_buff *skb) 2266 { 2267 return (TCP_SKB_CB(skb)->sacked & TCPCB_RETRANS) && 2268 tcp_tsopt_ecr_before(tp, tcp_skb_timestamp(skb)); 2269 } 2270 2271 /* Nothing was retransmitted or returned timestamp is less 2272 * than timestamp of the first retransmission. 2273 */ 2274 static inline bool tcp_packet_delayed(const struct tcp_sock *tp) 2275 { 2276 return !tp->retrans_stamp || 2277 tcp_tsopt_ecr_before(tp, tp->retrans_stamp); 2278 } 2279 2280 /* Undo procedures. */ 2281 2282 /* We can clear retrans_stamp when there are no retransmissions in the 2283 * window. It would seem that it is trivially available for us in 2284 * tp->retrans_out, however, that kind of assumptions doesn't consider 2285 * what will happen if errors occur when sending retransmission for the 2286 * second time. ...It could the that such segment has only 2287 * TCPCB_EVER_RETRANS set at the present time. It seems that checking 2288 * the head skb is enough except for some reneging corner cases that 2289 * are not worth the effort. 2290 * 2291 * Main reason for all this complexity is the fact that connection dying 2292 * time now depends on the validity of the retrans_stamp, in particular, 2293 * that successive retransmissions of a segment must not advance 2294 * retrans_stamp under any conditions. 2295 */ 2296 static bool tcp_any_retrans_done(const struct sock *sk) 2297 { 2298 const struct tcp_sock *tp = tcp_sk(sk); 2299 struct sk_buff *skb; 2300 2301 if (tp->retrans_out) 2302 return true; 2303 2304 skb = tcp_write_queue_head(sk); 2305 if (unlikely(skb && TCP_SKB_CB(skb)->sacked & TCPCB_EVER_RETRANS)) 2306 return true; 2307 2308 return false; 2309 } 2310 2311 #if FASTRETRANS_DEBUG > 1 2312 static void DBGUNDO(struct sock *sk, const char *msg) 2313 { 2314 struct tcp_sock *tp = tcp_sk(sk); 2315 struct inet_sock *inet = inet_sk(sk); 2316 2317 if (sk->sk_family == AF_INET) { 2318 pr_debug("Undo %s %pI4/%u c%u l%u ss%u/%u p%u\n", 2319 msg, 2320 &inet->inet_daddr, ntohs(inet->inet_dport), 2321 tp->snd_cwnd, tcp_left_out(tp), 2322 tp->snd_ssthresh, tp->prior_ssthresh, 2323 tp->packets_out); 2324 } 2325 #if IS_ENABLED(CONFIG_IPV6) 2326 else if (sk->sk_family == AF_INET6) { 2327 struct ipv6_pinfo *np = inet6_sk(sk); 2328 pr_debug("Undo %s %pI6/%u c%u l%u ss%u/%u p%u\n", 2329 msg, 2330 &np->daddr, ntohs(inet->inet_dport), 2331 tp->snd_cwnd, tcp_left_out(tp), 2332 tp->snd_ssthresh, tp->prior_ssthresh, 2333 tp->packets_out); 2334 } 2335 #endif 2336 } 2337 #else 2338 #define DBGUNDO(x...) do { } while (0) 2339 #endif 2340 2341 static void tcp_undo_cwnd_reduction(struct sock *sk, bool unmark_loss) 2342 { 2343 struct tcp_sock *tp = tcp_sk(sk); 2344 2345 if (unmark_loss) { 2346 struct sk_buff *skb; 2347 2348 tcp_for_write_queue(skb, sk) { 2349 if (skb == tcp_send_head(sk)) 2350 break; 2351 TCP_SKB_CB(skb)->sacked &= ~TCPCB_LOST; 2352 } 2353 tp->lost_out = 0; 2354 tcp_clear_all_retrans_hints(tp); 2355 } 2356 2357 if (tp->prior_ssthresh) { 2358 const struct inet_connection_sock *icsk = inet_csk(sk); 2359 2360 if (icsk->icsk_ca_ops->undo_cwnd) 2361 tp->snd_cwnd = icsk->icsk_ca_ops->undo_cwnd(sk); 2362 else 2363 tp->snd_cwnd = max(tp->snd_cwnd, tp->snd_ssthresh << 1); 2364 2365 if (tp->prior_ssthresh > tp->snd_ssthresh) { 2366 tp->snd_ssthresh = tp->prior_ssthresh; 2367 tcp_ecn_withdraw_cwr(tp); 2368 } 2369 } 2370 tp->snd_cwnd_stamp = tcp_time_stamp; 2371 tp->undo_marker = 0; 2372 } 2373 2374 static inline bool tcp_may_undo(const struct tcp_sock *tp) 2375 { 2376 return tp->undo_marker && (!tp->undo_retrans || tcp_packet_delayed(tp)); 2377 } 2378 2379 /* People celebrate: "We love our President!" */ 2380 static bool tcp_try_undo_recovery(struct sock *sk) 2381 { 2382 struct tcp_sock *tp = tcp_sk(sk); 2383 2384 if (tcp_may_undo(tp)) { 2385 int mib_idx; 2386 2387 /* Happy end! We did not retransmit anything 2388 * or our original transmission succeeded. 2389 */ 2390 DBGUNDO(sk, inet_csk(sk)->icsk_ca_state == TCP_CA_Loss ? "loss" : "retrans"); 2391 tcp_undo_cwnd_reduction(sk, false); 2392 if (inet_csk(sk)->icsk_ca_state == TCP_CA_Loss) 2393 mib_idx = LINUX_MIB_TCPLOSSUNDO; 2394 else 2395 mib_idx = LINUX_MIB_TCPFULLUNDO; 2396 2397 NET_INC_STATS_BH(sock_net(sk), mib_idx); 2398 } 2399 if (tp->snd_una == tp->high_seq && tcp_is_reno(tp)) { 2400 /* Hold old state until something *above* high_seq 2401 * is ACKed. For Reno it is MUST to prevent false 2402 * fast retransmits (RFC2582). SACK TCP is safe. */ 2403 tcp_moderate_cwnd(tp); 2404 if (!tcp_any_retrans_done(sk)) 2405 tp->retrans_stamp = 0; 2406 return true; 2407 } 2408 tcp_set_ca_state(sk, TCP_CA_Open); 2409 return false; 2410 } 2411 2412 /* Try to undo cwnd reduction, because D-SACKs acked all retransmitted data */ 2413 static bool tcp_try_undo_dsack(struct sock *sk) 2414 { 2415 struct tcp_sock *tp = tcp_sk(sk); 2416 2417 if (tp->undo_marker && !tp->undo_retrans) { 2418 DBGUNDO(sk, "D-SACK"); 2419 tcp_undo_cwnd_reduction(sk, false); 2420 NET_INC_STATS_BH(sock_net(sk), LINUX_MIB_TCPDSACKUNDO); 2421 return true; 2422 } 2423 return false; 2424 } 2425 2426 /* Undo during loss recovery after partial ACK or using F-RTO. */ 2427 static bool tcp_try_undo_loss(struct sock *sk, bool frto_undo) 2428 { 2429 struct tcp_sock *tp = tcp_sk(sk); 2430 2431 if (frto_undo || tcp_may_undo(tp)) { 2432 tcp_undo_cwnd_reduction(sk, true); 2433 2434 DBGUNDO(sk, "partial loss"); 2435 NET_INC_STATS_BH(sock_net(sk), LINUX_MIB_TCPLOSSUNDO); 2436 if (frto_undo) 2437 NET_INC_STATS_BH(sock_net(sk), 2438 LINUX_MIB_TCPSPURIOUSRTOS); 2439 inet_csk(sk)->icsk_retransmits = 0; 2440 if (frto_undo || tcp_is_sack(tp)) 2441 tcp_set_ca_state(sk, TCP_CA_Open); 2442 return true; 2443 } 2444 return false; 2445 } 2446 2447 /* The cwnd reduction in CWR and Recovery uses the PRR algorithm in RFC 6937. 2448 * It computes the number of packets to send (sndcnt) based on packets newly 2449 * delivered: 2450 * 1) If the packets in flight is larger than ssthresh, PRR spreads the 2451 * cwnd reductions across a full RTT. 2452 * 2) Otherwise PRR uses packet conservation to send as much as delivered. 2453 * But when the retransmits are acked without further losses, PRR 2454 * slow starts cwnd up to ssthresh to speed up the recovery. 2455 */ 2456 static void tcp_init_cwnd_reduction(struct sock *sk) 2457 { 2458 struct tcp_sock *tp = tcp_sk(sk); 2459 2460 tp->high_seq = tp->snd_nxt; 2461 tp->tlp_high_seq = 0; 2462 tp->snd_cwnd_cnt = 0; 2463 tp->prior_cwnd = tp->snd_cwnd; 2464 tp->prr_delivered = 0; 2465 tp->prr_out = 0; 2466 tp->snd_ssthresh = inet_csk(sk)->icsk_ca_ops->ssthresh(sk); 2467 tcp_ecn_queue_cwr(tp); 2468 } 2469 2470 static void tcp_cwnd_reduction(struct sock *sk, const int prior_unsacked, 2471 int fast_rexmit, int flag) 2472 { 2473 struct tcp_sock *tp = tcp_sk(sk); 2474 int sndcnt = 0; 2475 int delta = tp->snd_ssthresh - tcp_packets_in_flight(tp); 2476 int newly_acked_sacked = prior_unsacked - 2477 (tp->packets_out - tp->sacked_out); 2478 2479 if (newly_acked_sacked <= 0 || WARN_ON_ONCE(!tp->prior_cwnd)) 2480 return; 2481 2482 tp->prr_delivered += newly_acked_sacked; 2483 if (delta < 0) { 2484 u64 dividend = (u64)tp->snd_ssthresh * tp->prr_delivered + 2485 tp->prior_cwnd - 1; 2486 sndcnt = div_u64(dividend, tp->prior_cwnd) - tp->prr_out; 2487 } else if ((flag & FLAG_RETRANS_DATA_ACKED) && 2488 !(flag & FLAG_LOST_RETRANS)) { 2489 sndcnt = min_t(int, delta, 2490 max_t(int, tp->prr_delivered - tp->prr_out, 2491 newly_acked_sacked) + 1); 2492 } else { 2493 sndcnt = min(delta, newly_acked_sacked); 2494 } 2495 sndcnt = max(sndcnt, (fast_rexmit ? 1 : 0)); 2496 tp->snd_cwnd = tcp_packets_in_flight(tp) + sndcnt; 2497 } 2498 2499 static inline void tcp_end_cwnd_reduction(struct sock *sk) 2500 { 2501 struct tcp_sock *tp = tcp_sk(sk); 2502 2503 /* Reset cwnd to ssthresh in CWR or Recovery (unless it's undone) */ 2504 if (inet_csk(sk)->icsk_ca_state == TCP_CA_CWR || 2505 (tp->undo_marker && tp->snd_ssthresh < TCP_INFINITE_SSTHRESH)) { 2506 tp->snd_cwnd = tp->snd_ssthresh; 2507 tp->snd_cwnd_stamp = tcp_time_stamp; 2508 } 2509 tcp_ca_event(sk, CA_EVENT_COMPLETE_CWR); 2510 } 2511 2512 /* Enter CWR state. Disable cwnd undo since congestion is proven with ECN */ 2513 void tcp_enter_cwr(struct sock *sk) 2514 { 2515 struct tcp_sock *tp = tcp_sk(sk); 2516 2517 tp->prior_ssthresh = 0; 2518 if (inet_csk(sk)->icsk_ca_state < TCP_CA_CWR) { 2519 tp->undo_marker = 0; 2520 tcp_init_cwnd_reduction(sk); 2521 tcp_set_ca_state(sk, TCP_CA_CWR); 2522 } 2523 } 2524 EXPORT_SYMBOL(tcp_enter_cwr); 2525 2526 static void tcp_try_keep_open(struct sock *sk) 2527 { 2528 struct tcp_sock *tp = tcp_sk(sk); 2529 int state = TCP_CA_Open; 2530 2531 if (tcp_left_out(tp) || tcp_any_retrans_done(sk)) 2532 state = TCP_CA_Disorder; 2533 2534 if (inet_csk(sk)->icsk_ca_state != state) { 2535 tcp_set_ca_state(sk, state); 2536 tp->high_seq = tp->snd_nxt; 2537 } 2538 } 2539 2540 static void tcp_try_to_open(struct sock *sk, int flag, const int prior_unsacked) 2541 { 2542 struct tcp_sock *tp = tcp_sk(sk); 2543 2544 tcp_verify_left_out(tp); 2545 2546 if (!tcp_any_retrans_done(sk)) 2547 tp->retrans_stamp = 0; 2548 2549 if (flag & FLAG_ECE) 2550 tcp_enter_cwr(sk); 2551 2552 if (inet_csk(sk)->icsk_ca_state != TCP_CA_CWR) { 2553 tcp_try_keep_open(sk); 2554 } else { 2555 tcp_cwnd_reduction(sk, prior_unsacked, 0, flag); 2556 } 2557 } 2558 2559 static void tcp_mtup_probe_failed(struct sock *sk) 2560 { 2561 struct inet_connection_sock *icsk = inet_csk(sk); 2562 2563 icsk->icsk_mtup.search_high = icsk->icsk_mtup.probe_size - 1; 2564 icsk->icsk_mtup.probe_size = 0; 2565 NET_INC_STATS_BH(sock_net(sk), LINUX_MIB_TCPMTUPFAIL); 2566 } 2567 2568 static void tcp_mtup_probe_success(struct sock *sk) 2569 { 2570 struct tcp_sock *tp = tcp_sk(sk); 2571 struct inet_connection_sock *icsk = inet_csk(sk); 2572 2573 /* FIXME: breaks with very large cwnd */ 2574 tp->prior_ssthresh = tcp_current_ssthresh(sk); 2575 tp->snd_cwnd = tp->snd_cwnd * 2576 tcp_mss_to_mtu(sk, tp->mss_cache) / 2577 icsk->icsk_mtup.probe_size; 2578 tp->snd_cwnd_cnt = 0; 2579 tp->snd_cwnd_stamp = tcp_time_stamp; 2580 tp->snd_ssthresh = tcp_current_ssthresh(sk); 2581 2582 icsk->icsk_mtup.search_low = icsk->icsk_mtup.probe_size; 2583 icsk->icsk_mtup.probe_size = 0; 2584 tcp_sync_mss(sk, icsk->icsk_pmtu_cookie); 2585 NET_INC_STATS_BH(sock_net(sk), LINUX_MIB_TCPMTUPSUCCESS); 2586 } 2587 2588 /* Do a simple retransmit without using the backoff mechanisms in 2589 * tcp_timer. This is used for path mtu discovery. 2590 * The socket is already locked here. 2591 */ 2592 void tcp_simple_retransmit(struct sock *sk) 2593 { 2594 const struct inet_connection_sock *icsk = inet_csk(sk); 2595 struct tcp_sock *tp = tcp_sk(sk); 2596 struct sk_buff *skb; 2597 unsigned int mss = tcp_current_mss(sk); 2598 u32 prior_lost = tp->lost_out; 2599 2600 tcp_for_write_queue(skb, sk) { 2601 if (skb == tcp_send_head(sk)) 2602 break; 2603 if (tcp_skb_seglen(skb) > mss && 2604 !(TCP_SKB_CB(skb)->sacked & TCPCB_SACKED_ACKED)) { 2605 if (TCP_SKB_CB(skb)->sacked & TCPCB_SACKED_RETRANS) { 2606 TCP_SKB_CB(skb)->sacked &= ~TCPCB_SACKED_RETRANS; 2607 tp->retrans_out -= tcp_skb_pcount(skb); 2608 } 2609 tcp_skb_mark_lost_uncond_verify(tp, skb); 2610 } 2611 } 2612 2613 tcp_clear_retrans_hints_partial(tp); 2614 2615 if (prior_lost == tp->lost_out) 2616 return; 2617 2618 if (tcp_is_reno(tp)) 2619 tcp_limit_reno_sacked(tp); 2620 2621 tcp_verify_left_out(tp); 2622 2623 /* Don't muck with the congestion window here. 2624 * Reason is that we do not increase amount of _data_ 2625 * in network, but units changed and effective 2626 * cwnd/ssthresh really reduced now. 2627 */ 2628 if (icsk->icsk_ca_state != TCP_CA_Loss) { 2629 tp->high_seq = tp->snd_nxt; 2630 tp->snd_ssthresh = tcp_current_ssthresh(sk); 2631 tp->prior_ssthresh = 0; 2632 tp->undo_marker = 0; 2633 tcp_set_ca_state(sk, TCP_CA_Loss); 2634 } 2635 tcp_xmit_retransmit_queue(sk); 2636 } 2637 EXPORT_SYMBOL(tcp_simple_retransmit); 2638 2639 static void tcp_enter_recovery(struct sock *sk, bool ece_ack) 2640 { 2641 struct tcp_sock *tp = tcp_sk(sk); 2642 int mib_idx; 2643 2644 if (tcp_is_reno(tp)) 2645 mib_idx = LINUX_MIB_TCPRENORECOVERY; 2646 else 2647 mib_idx = LINUX_MIB_TCPSACKRECOVERY; 2648 2649 NET_INC_STATS_BH(sock_net(sk), mib_idx); 2650 2651 tp->prior_ssthresh = 0; 2652 tcp_init_undo(tp); 2653 2654 if (!tcp_in_cwnd_reduction(sk)) { 2655 if (!ece_ack) 2656 tp->prior_ssthresh = tcp_current_ssthresh(sk); 2657 tcp_init_cwnd_reduction(sk); 2658 } 2659 tcp_set_ca_state(sk, TCP_CA_Recovery); 2660 } 2661 2662 /* Process an ACK in CA_Loss state. Move to CA_Open if lost data are 2663 * recovered or spurious. Otherwise retransmits more on partial ACKs. 2664 */ 2665 static void tcp_process_loss(struct sock *sk, int flag, bool is_dupack) 2666 { 2667 struct tcp_sock *tp = tcp_sk(sk); 2668 bool recovered = !before(tp->snd_una, tp->high_seq); 2669 2670 if ((flag & FLAG_SND_UNA_ADVANCED) && 2671 tcp_try_undo_loss(sk, false)) 2672 return; 2673 2674 if (tp->frto) { /* F-RTO RFC5682 sec 3.1 (sack enhanced version). */ 2675 /* Step 3.b. A timeout is spurious if not all data are 2676 * lost, i.e., never-retransmitted data are (s)acked. 2677 */ 2678 if ((flag & FLAG_ORIG_SACK_ACKED) && 2679 tcp_try_undo_loss(sk, true)) 2680 return; 2681 2682 if (after(tp->snd_nxt, tp->high_seq)) { 2683 if (flag & FLAG_DATA_SACKED || is_dupack) 2684 tp->frto = 0; /* Step 3.a. loss was real */ 2685 } else if (flag & FLAG_SND_UNA_ADVANCED && !recovered) { 2686 tp->high_seq = tp->snd_nxt; 2687 __tcp_push_pending_frames(sk, tcp_current_mss(sk), 2688 TCP_NAGLE_OFF); 2689 if (after(tp->snd_nxt, tp->high_seq)) 2690 return; /* Step 2.b */ 2691 tp->frto = 0; 2692 } 2693 } 2694 2695 if (recovered) { 2696 /* F-RTO RFC5682 sec 3.1 step 2.a and 1st part of step 3.a */ 2697 tcp_try_undo_recovery(sk); 2698 return; 2699 } 2700 if (tcp_is_reno(tp)) { 2701 /* A Reno DUPACK means new data in F-RTO step 2.b above are 2702 * delivered. Lower inflight to clock out (re)tranmissions. 2703 */ 2704 if (after(tp->snd_nxt, tp->high_seq) && is_dupack) 2705 tcp_add_reno_sack(sk); 2706 else if (flag & FLAG_SND_UNA_ADVANCED) 2707 tcp_reset_reno_sack(tp); 2708 } 2709 tcp_xmit_retransmit_queue(sk); 2710 } 2711 2712 /* Undo during fast recovery after partial ACK. */ 2713 static bool tcp_try_undo_partial(struct sock *sk, const int acked, 2714 const int prior_unsacked, int flag) 2715 { 2716 struct tcp_sock *tp = tcp_sk(sk); 2717 2718 if (tp->undo_marker && tcp_packet_delayed(tp)) { 2719 /* Plain luck! Hole if filled with delayed 2720 * packet, rather than with a retransmit. 2721 */ 2722 tcp_update_reordering(sk, tcp_fackets_out(tp) + acked, 1); 2723 2724 /* We are getting evidence that the reordering degree is higher 2725 * than we realized. If there are no retransmits out then we 2726 * can undo. Otherwise we clock out new packets but do not 2727 * mark more packets lost or retransmit more. 2728 */ 2729 if (tp->retrans_out) { 2730 tcp_cwnd_reduction(sk, prior_unsacked, 0, flag); 2731 return true; 2732 } 2733 2734 if (!tcp_any_retrans_done(sk)) 2735 tp->retrans_stamp = 0; 2736 2737 DBGUNDO(sk, "partial recovery"); 2738 tcp_undo_cwnd_reduction(sk, true); 2739 NET_INC_STATS_BH(sock_net(sk), LINUX_MIB_TCPPARTIALUNDO); 2740 tcp_try_keep_open(sk); 2741 return true; 2742 } 2743 return false; 2744 } 2745 2746 /* Process an event, which can update packets-in-flight not trivially. 2747 * Main goal of this function is to calculate new estimate for left_out, 2748 * taking into account both packets sitting in receiver's buffer and 2749 * packets lost by network. 2750 * 2751 * Besides that it does CWND reduction, when packet loss is detected 2752 * and changes state of machine. 2753 * 2754 * It does _not_ decide what to send, it is made in function 2755 * tcp_xmit_retransmit_queue(). 2756 */ 2757 static void tcp_fastretrans_alert(struct sock *sk, const int acked, 2758 const int prior_unsacked, 2759 bool is_dupack, int flag) 2760 { 2761 struct inet_connection_sock *icsk = inet_csk(sk); 2762 struct tcp_sock *tp = tcp_sk(sk); 2763 bool do_lost = is_dupack || ((flag & FLAG_DATA_SACKED) && 2764 (tcp_fackets_out(tp) > tp->reordering)); 2765 int fast_rexmit = 0; 2766 2767 if (WARN_ON(!tp->packets_out && tp->sacked_out)) 2768 tp->sacked_out = 0; 2769 if (WARN_ON(!tp->sacked_out && tp->fackets_out)) 2770 tp->fackets_out = 0; 2771 2772 /* Now state machine starts. 2773 * A. ECE, hence prohibit cwnd undoing, the reduction is required. */ 2774 if (flag & FLAG_ECE) 2775 tp->prior_ssthresh = 0; 2776 2777 /* B. In all the states check for reneging SACKs. */ 2778 if (tcp_check_sack_reneging(sk, flag)) 2779 return; 2780 2781 /* C. Check consistency of the current state. */ 2782 tcp_verify_left_out(tp); 2783 2784 /* D. Check state exit conditions. State can be terminated 2785 * when high_seq is ACKed. */ 2786 if (icsk->icsk_ca_state == TCP_CA_Open) { 2787 WARN_ON(tp->retrans_out != 0); 2788 tp->retrans_stamp = 0; 2789 } else if (!before(tp->snd_una, tp->high_seq)) { 2790 switch (icsk->icsk_ca_state) { 2791 case TCP_CA_CWR: 2792 /* CWR is to be held something *above* high_seq 2793 * is ACKed for CWR bit to reach receiver. */ 2794 if (tp->snd_una != tp->high_seq) { 2795 tcp_end_cwnd_reduction(sk); 2796 tcp_set_ca_state(sk, TCP_CA_Open); 2797 } 2798 break; 2799 2800 case TCP_CA_Recovery: 2801 if (tcp_is_reno(tp)) 2802 tcp_reset_reno_sack(tp); 2803 if (tcp_try_undo_recovery(sk)) 2804 return; 2805 tcp_end_cwnd_reduction(sk); 2806 break; 2807 } 2808 } 2809 2810 /* Use RACK to detect loss */ 2811 if (sysctl_tcp_recovery & TCP_RACK_LOST_RETRANS && 2812 tcp_rack_mark_lost(sk)) 2813 flag |= FLAG_LOST_RETRANS; 2814 2815 /* E. Process state. */ 2816 switch (icsk->icsk_ca_state) { 2817 case TCP_CA_Recovery: 2818 if (!(flag & FLAG_SND_UNA_ADVANCED)) { 2819 if (tcp_is_reno(tp) && is_dupack) 2820 tcp_add_reno_sack(sk); 2821 } else { 2822 if (tcp_try_undo_partial(sk, acked, prior_unsacked, flag)) 2823 return; 2824 /* Partial ACK arrived. Force fast retransmit. */ 2825 do_lost = tcp_is_reno(tp) || 2826 tcp_fackets_out(tp) > tp->reordering; 2827 } 2828 if (tcp_try_undo_dsack(sk)) { 2829 tcp_try_keep_open(sk); 2830 return; 2831 } 2832 break; 2833 case TCP_CA_Loss: 2834 tcp_process_loss(sk, flag, is_dupack); 2835 if (icsk->icsk_ca_state != TCP_CA_Open && 2836 !(flag & FLAG_LOST_RETRANS)) 2837 return; 2838 /* Change state if cwnd is undone or retransmits are lost */ 2839 default: 2840 if (tcp_is_reno(tp)) { 2841 if (flag & FLAG_SND_UNA_ADVANCED) 2842 tcp_reset_reno_sack(tp); 2843 if (is_dupack) 2844 tcp_add_reno_sack(sk); 2845 } 2846 2847 if (icsk->icsk_ca_state <= TCP_CA_Disorder) 2848 tcp_try_undo_dsack(sk); 2849 2850 if (!tcp_time_to_recover(sk, flag)) { 2851 tcp_try_to_open(sk, flag, prior_unsacked); 2852 return; 2853 } 2854 2855 /* MTU probe failure: don't reduce cwnd */ 2856 if (icsk->icsk_ca_state < TCP_CA_CWR && 2857 icsk->icsk_mtup.probe_size && 2858 tp->snd_una == tp->mtu_probe.probe_seq_start) { 2859 tcp_mtup_probe_failed(sk); 2860 /* Restores the reduction we did in tcp_mtup_probe() */ 2861 tp->snd_cwnd++; 2862 tcp_simple_retransmit(sk); 2863 return; 2864 } 2865 2866 /* Otherwise enter Recovery state */ 2867 tcp_enter_recovery(sk, (flag & FLAG_ECE)); 2868 fast_rexmit = 1; 2869 } 2870 2871 if (do_lost) 2872 tcp_update_scoreboard(sk, fast_rexmit); 2873 tcp_cwnd_reduction(sk, prior_unsacked, fast_rexmit, flag); 2874 tcp_xmit_retransmit_queue(sk); 2875 } 2876 2877 /* Kathleen Nichols' algorithm for tracking the minimum value of 2878 * a data stream over some fixed time interval. (E.g., the minimum 2879 * RTT over the past five minutes.) It uses constant space and constant 2880 * time per update yet almost always delivers the same minimum as an 2881 * implementation that has to keep all the data in the window. 2882 * 2883 * The algorithm keeps track of the best, 2nd best & 3rd best min 2884 * values, maintaining an invariant that the measurement time of the 2885 * n'th best >= n-1'th best. It also makes sure that the three values 2886 * are widely separated in the time window since that bounds the worse 2887 * case error when that data is monotonically increasing over the window. 2888 * 2889 * Upon getting a new min, we can forget everything earlier because it 2890 * has no value - the new min is <= everything else in the window by 2891 * definition and it's the most recent. So we restart fresh on every new min 2892 * and overwrites 2nd & 3rd choices. The same property holds for 2nd & 3rd 2893 * best. 2894 */ 2895 static void tcp_update_rtt_min(struct sock *sk, u32 rtt_us) 2896 { 2897 const u32 now = tcp_time_stamp, wlen = sysctl_tcp_min_rtt_wlen * HZ; 2898 struct rtt_meas *m = tcp_sk(sk)->rtt_min; 2899 struct rtt_meas rttm = { 2900 .rtt = likely(rtt_us) ? rtt_us : jiffies_to_usecs(1), 2901 .ts = now, 2902 }; 2903 u32 elapsed; 2904 2905 /* Check if the new measurement updates the 1st, 2nd, or 3rd choices */ 2906 if (unlikely(rttm.rtt <= m[0].rtt)) 2907 m[0] = m[1] = m[2] = rttm; 2908 else if (rttm.rtt <= m[1].rtt) 2909 m[1] = m[2] = rttm; 2910 else if (rttm.rtt <= m[2].rtt) 2911 m[2] = rttm; 2912 2913 elapsed = now - m[0].ts; 2914 if (unlikely(elapsed > wlen)) { 2915 /* Passed entire window without a new min so make 2nd choice 2916 * the new min & 3rd choice the new 2nd. So forth and so on. 2917 */ 2918 m[0] = m[1]; 2919 m[1] = m[2]; 2920 m[2] = rttm; 2921 if (now - m[0].ts > wlen) { 2922 m[0] = m[1]; 2923 m[1] = rttm; 2924 if (now - m[0].ts > wlen) 2925 m[0] = rttm; 2926 } 2927 } else if (m[1].ts == m[0].ts && elapsed > wlen / 4) { 2928 /* Passed a quarter of the window without a new min so 2929 * take 2nd choice from the 2nd quarter of the window. 2930 */ 2931 m[2] = m[1] = rttm; 2932 } else if (m[2].ts == m[1].ts && elapsed > wlen / 2) { 2933 /* Passed half the window without a new min so take the 3rd 2934 * choice from the last half of the window. 2935 */ 2936 m[2] = rttm; 2937 } 2938 } 2939 2940 static inline bool tcp_ack_update_rtt(struct sock *sk, const int flag, 2941 long seq_rtt_us, long sack_rtt_us, 2942 long ca_rtt_us) 2943 { 2944 const struct tcp_sock *tp = tcp_sk(sk); 2945 2946 /* Prefer RTT measured from ACK's timing to TS-ECR. This is because 2947 * broken middle-boxes or peers may corrupt TS-ECR fields. But 2948 * Karn's algorithm forbids taking RTT if some retransmitted data 2949 * is acked (RFC6298). 2950 */ 2951 if (seq_rtt_us < 0) 2952 seq_rtt_us = sack_rtt_us; 2953 2954 /* RTTM Rule: A TSecr value received in a segment is used to 2955 * update the averaged RTT measurement only if the segment 2956 * acknowledges some new data, i.e., only if it advances the 2957 * left edge of the send window. 2958 * See draft-ietf-tcplw-high-performance-00, section 3.3. 2959 */ 2960 if (seq_rtt_us < 0 && tp->rx_opt.saw_tstamp && tp->rx_opt.rcv_tsecr && 2961 flag & FLAG_ACKED) 2962 seq_rtt_us = ca_rtt_us = jiffies_to_usecs(tcp_time_stamp - 2963 tp->rx_opt.rcv_tsecr); 2964 if (seq_rtt_us < 0) 2965 return false; 2966 2967 /* ca_rtt_us >= 0 is counting on the invariant that ca_rtt_us is 2968 * always taken together with ACK, SACK, or TS-opts. Any negative 2969 * values will be skipped with the seq_rtt_us < 0 check above. 2970 */ 2971 tcp_update_rtt_min(sk, ca_rtt_us); 2972 tcp_rtt_estimator(sk, seq_rtt_us); 2973 tcp_set_rto(sk); 2974 2975 /* RFC6298: only reset backoff on valid RTT measurement. */ 2976 inet_csk(sk)->icsk_backoff = 0; 2977 return true; 2978 } 2979 2980 /* Compute time elapsed between (last) SYNACK and the ACK completing 3WHS. */ 2981 void tcp_synack_rtt_meas(struct sock *sk, struct request_sock *req) 2982 { 2983 long rtt_us = -1L; 2984 2985 if (req && !req->num_retrans && tcp_rsk(req)->snt_synack.v64) { 2986 struct skb_mstamp now; 2987 2988 skb_mstamp_get(&now); 2989 rtt_us = skb_mstamp_us_delta(&now, &tcp_rsk(req)->snt_synack); 2990 } 2991 2992 tcp_ack_update_rtt(sk, FLAG_SYN_ACKED, rtt_us, -1L, rtt_us); 2993 } 2994 2995 2996 static void tcp_cong_avoid(struct sock *sk, u32 ack, u32 acked) 2997 { 2998 const struct inet_connection_sock *icsk = inet_csk(sk); 2999 3000 icsk->icsk_ca_ops->cong_avoid(sk, ack, acked); 3001 tcp_sk(sk)->snd_cwnd_stamp = tcp_time_stamp; 3002 } 3003 3004 /* Restart timer after forward progress on connection. 3005 * RFC2988 recommends to restart timer to now+rto. 3006 */ 3007 void tcp_rearm_rto(struct sock *sk) 3008 { 3009 const struct inet_connection_sock *icsk = inet_csk(sk); 3010 struct tcp_sock *tp = tcp_sk(sk); 3011 3012 /* If the retrans timer is currently being used by Fast Open 3013 * for SYN-ACK retrans purpose, stay put. 3014 */ 3015 if (tp->fastopen_rsk) 3016 return; 3017 3018 if (!tp->packets_out) { 3019 inet_csk_clear_xmit_timer(sk, ICSK_TIME_RETRANS); 3020 } else { 3021 u32 rto = inet_csk(sk)->icsk_rto; 3022 /* Offset the time elapsed after installing regular RTO */ 3023 if (icsk->icsk_pending == ICSK_TIME_EARLY_RETRANS || 3024 icsk->icsk_pending == ICSK_TIME_LOSS_PROBE) { 3025 struct sk_buff *skb = tcp_write_queue_head(sk); 3026 const u32 rto_time_stamp = 3027 tcp_skb_timestamp(skb) + rto; 3028 s32 delta = (s32)(rto_time_stamp - tcp_time_stamp); 3029 /* delta may not be positive if the socket is locked 3030 * when the retrans timer fires and is rescheduled. 3031 */ 3032 if (delta > 0) 3033 rto = delta; 3034 } 3035 inet_csk_reset_xmit_timer(sk, ICSK_TIME_RETRANS, rto, 3036 TCP_RTO_MAX); 3037 } 3038 } 3039 3040 /* This function is called when the delayed ER timer fires. TCP enters 3041 * fast recovery and performs fast-retransmit. 3042 */ 3043 void tcp_resume_early_retransmit(struct sock *sk) 3044 { 3045 struct tcp_sock *tp = tcp_sk(sk); 3046 3047 tcp_rearm_rto(sk); 3048 3049 /* Stop if ER is disabled after the delayed ER timer is scheduled */ 3050 if (!tp->do_early_retrans) 3051 return; 3052 3053 tcp_enter_recovery(sk, false); 3054 tcp_update_scoreboard(sk, 1); 3055 tcp_xmit_retransmit_queue(sk); 3056 } 3057 3058 /* If we get here, the whole TSO packet has not been acked. */ 3059 static u32 tcp_tso_acked(struct sock *sk, struct sk_buff *skb) 3060 { 3061 struct tcp_sock *tp = tcp_sk(sk); 3062 u32 packets_acked; 3063 3064 BUG_ON(!after(TCP_SKB_CB(skb)->end_seq, tp->snd_una)); 3065 3066 packets_acked = tcp_skb_pcount(skb); 3067 if (tcp_trim_head(sk, skb, tp->snd_una - TCP_SKB_CB(skb)->seq)) 3068 return 0; 3069 packets_acked -= tcp_skb_pcount(skb); 3070 3071 if (packets_acked) { 3072 BUG_ON(tcp_skb_pcount(skb) == 0); 3073 BUG_ON(!before(TCP_SKB_CB(skb)->seq, TCP_SKB_CB(skb)->end_seq)); 3074 } 3075 3076 return packets_acked; 3077 } 3078 3079 static void tcp_ack_tstamp(struct sock *sk, struct sk_buff *skb, 3080 u32 prior_snd_una) 3081 { 3082 const struct skb_shared_info *shinfo; 3083 3084 /* Avoid cache line misses to get skb_shinfo() and shinfo->tx_flags */ 3085 if (likely(!(sk->sk_tsflags & SOF_TIMESTAMPING_TX_ACK))) 3086 return; 3087 3088 shinfo = skb_shinfo(skb); 3089 if ((shinfo->tx_flags & SKBTX_ACK_TSTAMP) && 3090 between(shinfo->tskey, prior_snd_una, tcp_sk(sk)->snd_una - 1)) 3091 __skb_tstamp_tx(skb, NULL, sk, SCM_TSTAMP_ACK); 3092 } 3093 3094 /* Remove acknowledged frames from the retransmission queue. If our packet 3095 * is before the ack sequence we can discard it as it's confirmed to have 3096 * arrived at the other end. 3097 */ 3098 static int tcp_clean_rtx_queue(struct sock *sk, int prior_fackets, 3099 u32 prior_snd_una, 3100 struct tcp_sacktag_state *sack) 3101 { 3102 const struct inet_connection_sock *icsk = inet_csk(sk); 3103 struct skb_mstamp first_ackt, last_ackt, now; 3104 struct tcp_sock *tp = tcp_sk(sk); 3105 u32 prior_sacked = tp->sacked_out; 3106 u32 reord = tp->packets_out; 3107 bool fully_acked = true; 3108 long sack_rtt_us = -1L; 3109 long seq_rtt_us = -1L; 3110 long ca_rtt_us = -1L; 3111 struct sk_buff *skb; 3112 u32 pkts_acked = 0; 3113 bool rtt_update; 3114 int flag = 0; 3115 3116 first_ackt.v64 = 0; 3117 3118 while ((skb = tcp_write_queue_head(sk)) && skb != tcp_send_head(sk)) { 3119 struct tcp_skb_cb *scb = TCP_SKB_CB(skb); 3120 u8 sacked = scb->sacked; 3121 u32 acked_pcount; 3122 3123 tcp_ack_tstamp(sk, skb, prior_snd_una); 3124 3125 /* Determine how many packets and what bytes were acked, tso and else */ 3126 if (after(scb->end_seq, tp->snd_una)) { 3127 if (tcp_skb_pcount(skb) == 1 || 3128 !after(tp->snd_una, scb->seq)) 3129 break; 3130 3131 acked_pcount = tcp_tso_acked(sk, skb); 3132 if (!acked_pcount) 3133 break; 3134 3135 fully_acked = false; 3136 } else { 3137 /* Speedup tcp_unlink_write_queue() and next loop */ 3138 prefetchw(skb->next); 3139 acked_pcount = tcp_skb_pcount(skb); 3140 } 3141 3142 if (unlikely(sacked & TCPCB_RETRANS)) { 3143 if (sacked & TCPCB_SACKED_RETRANS) 3144 tp->retrans_out -= acked_pcount; 3145 flag |= FLAG_RETRANS_DATA_ACKED; 3146 } else if (!(sacked & TCPCB_SACKED_ACKED)) { 3147 last_ackt = skb->skb_mstamp; 3148 WARN_ON_ONCE(last_ackt.v64 == 0); 3149 if (!first_ackt.v64) 3150 first_ackt = last_ackt; 3151 3152 reord = min(pkts_acked, reord); 3153 if (!after(scb->end_seq, tp->high_seq)) 3154 flag |= FLAG_ORIG_SACK_ACKED; 3155 } 3156 3157 if (sacked & TCPCB_SACKED_ACKED) 3158 tp->sacked_out -= acked_pcount; 3159 else if (tcp_is_sack(tp) && !tcp_skb_spurious_retrans(tp, skb)) 3160 tcp_rack_advance(tp, &skb->skb_mstamp, sacked); 3161 if (sacked & TCPCB_LOST) 3162 tp->lost_out -= acked_pcount; 3163 3164 tp->packets_out -= acked_pcount; 3165 pkts_acked += acked_pcount; 3166 3167 /* Initial outgoing SYN's get put onto the write_queue 3168 * just like anything else we transmit. It is not 3169 * true data, and if we misinform our callers that 3170 * this ACK acks real data, we will erroneously exit 3171 * connection startup slow start one packet too 3172 * quickly. This is severely frowned upon behavior. 3173 */ 3174 if (likely(!(scb->tcp_flags & TCPHDR_SYN))) { 3175 flag |= FLAG_DATA_ACKED; 3176 } else { 3177 flag |= FLAG_SYN_ACKED; 3178 tp->retrans_stamp = 0; 3179 } 3180 3181 if (!fully_acked) 3182 break; 3183 3184 tcp_unlink_write_queue(skb, sk); 3185 sk_wmem_free_skb(sk, skb); 3186 if (unlikely(skb == tp->retransmit_skb_hint)) 3187 tp->retransmit_skb_hint = NULL; 3188 if (unlikely(skb == tp->lost_skb_hint)) 3189 tp->lost_skb_hint = NULL; 3190 } 3191 3192 if (likely(between(tp->snd_up, prior_snd_una, tp->snd_una))) 3193 tp->snd_up = tp->snd_una; 3194 3195 if (skb && (TCP_SKB_CB(skb)->sacked & TCPCB_SACKED_ACKED)) 3196 flag |= FLAG_SACK_RENEGING; 3197 3198 skb_mstamp_get(&now); 3199 if (likely(first_ackt.v64) && !(flag & FLAG_RETRANS_DATA_ACKED)) { 3200 seq_rtt_us = skb_mstamp_us_delta(&now, &first_ackt); 3201 ca_rtt_us = skb_mstamp_us_delta(&now, &last_ackt); 3202 } 3203 if (sack->first_sackt.v64) { 3204 sack_rtt_us = skb_mstamp_us_delta(&now, &sack->first_sackt); 3205 ca_rtt_us = skb_mstamp_us_delta(&now, &sack->last_sackt); 3206 } 3207 3208 rtt_update = tcp_ack_update_rtt(sk, flag, seq_rtt_us, sack_rtt_us, 3209 ca_rtt_us); 3210 3211 if (flag & FLAG_ACKED) { 3212 tcp_rearm_rto(sk); 3213 if (unlikely(icsk->icsk_mtup.probe_size && 3214 !after(tp->mtu_probe.probe_seq_end, tp->snd_una))) { 3215 tcp_mtup_probe_success(sk); 3216 } 3217 3218 if (tcp_is_reno(tp)) { 3219 tcp_remove_reno_sacks(sk, pkts_acked); 3220 } else { 3221 int delta; 3222 3223 /* Non-retransmitted hole got filled? That's reordering */ 3224 if (reord < prior_fackets) 3225 tcp_update_reordering(sk, tp->fackets_out - reord, 0); 3226 3227 delta = tcp_is_fack(tp) ? pkts_acked : 3228 prior_sacked - tp->sacked_out; 3229 tp->lost_cnt_hint -= min(tp->lost_cnt_hint, delta); 3230 } 3231 3232 tp->fackets_out -= min(pkts_acked, tp->fackets_out); 3233 3234 } else if (skb && rtt_update && sack_rtt_us >= 0 && 3235 sack_rtt_us > skb_mstamp_us_delta(&now, &skb->skb_mstamp)) { 3236 /* Do not re-arm RTO if the sack RTT is measured from data sent 3237 * after when the head was last (re)transmitted. Otherwise the 3238 * timeout may continue to extend in loss recovery. 3239 */ 3240 tcp_rearm_rto(sk); 3241 } 3242 3243 if (icsk->icsk_ca_ops->pkts_acked) 3244 icsk->icsk_ca_ops->pkts_acked(sk, pkts_acked, ca_rtt_us); 3245 3246 #if FASTRETRANS_DEBUG > 0 3247 WARN_ON((int)tp->sacked_out < 0); 3248 WARN_ON((int)tp->lost_out < 0); 3249 WARN_ON((int)tp->retrans_out < 0); 3250 if (!tp->packets_out && tcp_is_sack(tp)) { 3251 icsk = inet_csk(sk); 3252 if (tp->lost_out) { 3253 pr_debug("Leak l=%u %d\n", 3254 tp->lost_out, icsk->icsk_ca_state); 3255 tp->lost_out = 0; 3256 } 3257 if (tp->sacked_out) { 3258 pr_debug("Leak s=%u %d\n", 3259 tp->sacked_out, icsk->icsk_ca_state); 3260 tp->sacked_out = 0; 3261 } 3262 if (tp->retrans_out) { 3263 pr_debug("Leak r=%u %d\n", 3264 tp->retrans_out, icsk->icsk_ca_state); 3265 tp->retrans_out = 0; 3266 } 3267 } 3268 #endif 3269 return flag; 3270 } 3271 3272 static void tcp_ack_probe(struct sock *sk) 3273 { 3274 const struct tcp_sock *tp = tcp_sk(sk); 3275 struct inet_connection_sock *icsk = inet_csk(sk); 3276 3277 /* Was it a usable window open? */ 3278 3279 if (!after(TCP_SKB_CB(tcp_send_head(sk))->end_seq, tcp_wnd_end(tp))) { 3280 icsk->icsk_backoff = 0; 3281 inet_csk_clear_xmit_timer(sk, ICSK_TIME_PROBE0); 3282 /* Socket must be waked up by subsequent tcp_data_snd_check(). 3283 * This function is not for random using! 3284 */ 3285 } else { 3286 unsigned long when = tcp_probe0_when(sk, TCP_RTO_MAX); 3287 3288 inet_csk_reset_xmit_timer(sk, ICSK_TIME_PROBE0, 3289 when, TCP_RTO_MAX); 3290 } 3291 } 3292 3293 static inline bool tcp_ack_is_dubious(const struct sock *sk, const int flag) 3294 { 3295 return !(flag & FLAG_NOT_DUP) || (flag & FLAG_CA_ALERT) || 3296 inet_csk(sk)->icsk_ca_state != TCP_CA_Open; 3297 } 3298 3299 /* Decide wheather to run the increase function of congestion control. */ 3300 static inline bool tcp_may_raise_cwnd(const struct sock *sk, const int flag) 3301 { 3302 if (tcp_in_cwnd_reduction(sk)) 3303 return false; 3304 3305 /* If reordering is high then always grow cwnd whenever data is 3306 * delivered regardless of its ordering. Otherwise stay conservative 3307 * and only grow cwnd on in-order delivery (RFC5681). A stretched ACK w/ 3308 * new SACK or ECE mark may first advance cwnd here and later reduce 3309 * cwnd in tcp_fastretrans_alert() based on more states. 3310 */ 3311 if (tcp_sk(sk)->reordering > sysctl_tcp_reordering) 3312 return flag & FLAG_FORWARD_PROGRESS; 3313 3314 return flag & FLAG_DATA_ACKED; 3315 } 3316 3317 /* Check that window update is acceptable. 3318 * The function assumes that snd_una<=ack<=snd_next. 3319 */ 3320 static inline bool tcp_may_update_window(const struct tcp_sock *tp, 3321 const u32 ack, const u32 ack_seq, 3322 const u32 nwin) 3323 { 3324 return after(ack, tp->snd_una) || 3325 after(ack_seq, tp->snd_wl1) || 3326 (ack_seq == tp->snd_wl1 && nwin > tp->snd_wnd); 3327 } 3328 3329 /* If we update tp->snd_una, also update tp->bytes_acked */ 3330 static void tcp_snd_una_update(struct tcp_sock *tp, u32 ack) 3331 { 3332 u32 delta = ack - tp->snd_una; 3333 3334 u64_stats_update_begin(&tp->syncp); 3335 tp->bytes_acked += delta; 3336 u64_stats_update_end(&tp->syncp); 3337 tp->snd_una = ack; 3338 } 3339 3340 /* If we update tp->rcv_nxt, also update tp->bytes_received */ 3341 static void tcp_rcv_nxt_update(struct tcp_sock *tp, u32 seq) 3342 { 3343 u32 delta = seq - tp->rcv_nxt; 3344 3345 u64_stats_update_begin(&tp->syncp); 3346 tp->bytes_received += delta; 3347 u64_stats_update_end(&tp->syncp); 3348 tp->rcv_nxt = seq; 3349 } 3350 3351 /* Update our send window. 3352 * 3353 * Window update algorithm, described in RFC793/RFC1122 (used in linux-2.2 3354 * and in FreeBSD. NetBSD's one is even worse.) is wrong. 3355 */ 3356 static int tcp_ack_update_window(struct sock *sk, const struct sk_buff *skb, u32 ack, 3357 u32 ack_seq) 3358 { 3359 struct tcp_sock *tp = tcp_sk(sk); 3360 int flag = 0; 3361 u32 nwin = ntohs(tcp_hdr(skb)->window); 3362 3363 if (likely(!tcp_hdr(skb)->syn)) 3364 nwin <<= tp->rx_opt.snd_wscale; 3365 3366 if (tcp_may_update_window(tp, ack, ack_seq, nwin)) { 3367 flag |= FLAG_WIN_UPDATE; 3368 tcp_update_wl(tp, ack_seq); 3369 3370 if (tp->snd_wnd != nwin) { 3371 tp->snd_wnd = nwin; 3372 3373 /* Note, it is the only place, where 3374 * fast path is recovered for sending TCP. 3375 */ 3376 tp->pred_flags = 0; 3377 tcp_fast_path_check(sk); 3378 3379 if (tcp_send_head(sk)) 3380 tcp_slow_start_after_idle_check(sk); 3381 3382 if (nwin > tp->max_window) { 3383 tp->max_window = nwin; 3384 tcp_sync_mss(sk, inet_csk(sk)->icsk_pmtu_cookie); 3385 } 3386 } 3387 } 3388 3389 tcp_snd_una_update(tp, ack); 3390 3391 return flag; 3392 } 3393 3394 /* Return true if we're currently rate-limiting out-of-window ACKs and 3395 * thus shouldn't send a dupack right now. We rate-limit dupacks in 3396 * response to out-of-window SYNs or ACKs to mitigate ACK loops or DoS 3397 * attacks that send repeated SYNs or ACKs for the same connection. To 3398 * do this, we do not send a duplicate SYNACK or ACK if the remote 3399 * endpoint is sending out-of-window SYNs or pure ACKs at a high rate. 3400 */ 3401 bool tcp_oow_rate_limited(struct net *net, const struct sk_buff *skb, 3402 int mib_idx, u32 *last_oow_ack_time) 3403 { 3404 /* Data packets without SYNs are not likely part of an ACK loop. */ 3405 if ((TCP_SKB_CB(skb)->seq != TCP_SKB_CB(skb)->end_seq) && 3406 !tcp_hdr(skb)->syn) 3407 goto not_rate_limited; 3408 3409 if (*last_oow_ack_time) { 3410 s32 elapsed = (s32)(tcp_time_stamp - *last_oow_ack_time); 3411 3412 if (0 <= elapsed && elapsed < sysctl_tcp_invalid_ratelimit) { 3413 NET_INC_STATS_BH(net, mib_idx); 3414 return true; /* rate-limited: don't send yet! */ 3415 } 3416 } 3417 3418 *last_oow_ack_time = tcp_time_stamp; 3419 3420 not_rate_limited: 3421 return false; /* not rate-limited: go ahead, send dupack now! */ 3422 } 3423 3424 /* RFC 5961 7 [ACK Throttling] */ 3425 static void tcp_send_challenge_ack(struct sock *sk, const struct sk_buff *skb) 3426 { 3427 /* unprotected vars, we dont care of overwrites */ 3428 static u32 challenge_timestamp; 3429 static unsigned int challenge_count; 3430 struct tcp_sock *tp = tcp_sk(sk); 3431 u32 now; 3432 3433 /* First check our per-socket dupack rate limit. */ 3434 if (tcp_oow_rate_limited(sock_net(sk), skb, 3435 LINUX_MIB_TCPACKSKIPPEDCHALLENGE, 3436 &tp->last_oow_ack_time)) 3437 return; 3438 3439 /* Then check the check host-wide RFC 5961 rate limit. */ 3440 now = jiffies / HZ; 3441 if (now != challenge_timestamp) { 3442 challenge_timestamp = now; 3443 challenge_count = 0; 3444 } 3445 if (++challenge_count <= sysctl_tcp_challenge_ack_limit) { 3446 NET_INC_STATS_BH(sock_net(sk), LINUX_MIB_TCPCHALLENGEACK); 3447 tcp_send_ack(sk); 3448 } 3449 } 3450 3451 static void tcp_store_ts_recent(struct tcp_sock *tp) 3452 { 3453 tp->rx_opt.ts_recent = tp->rx_opt.rcv_tsval; 3454 tp->rx_opt.ts_recent_stamp = get_seconds(); 3455 } 3456 3457 static void tcp_replace_ts_recent(struct tcp_sock *tp, u32 seq) 3458 { 3459 if (tp->rx_opt.saw_tstamp && !after(seq, tp->rcv_wup)) { 3460 /* PAWS bug workaround wrt. ACK frames, the PAWS discard 3461 * extra check below makes sure this can only happen 3462 * for pure ACK frames. -DaveM 3463 * 3464 * Not only, also it occurs for expired timestamps. 3465 */ 3466 3467 if (tcp_paws_check(&tp->rx_opt, 0)) 3468 tcp_store_ts_recent(tp); 3469 } 3470 } 3471 3472 /* This routine deals with acks during a TLP episode. 3473 * We mark the end of a TLP episode on receiving TLP dupack or when 3474 * ack is after tlp_high_seq. 3475 * Ref: loss detection algorithm in draft-dukkipati-tcpm-tcp-loss-probe. 3476 */ 3477 static void tcp_process_tlp_ack(struct sock *sk, u32 ack, int flag) 3478 { 3479 struct tcp_sock *tp = tcp_sk(sk); 3480 3481 if (before(ack, tp->tlp_high_seq)) 3482 return; 3483 3484 if (flag & FLAG_DSACKING_ACK) { 3485 /* This DSACK means original and TLP probe arrived; no loss */ 3486 tp->tlp_high_seq = 0; 3487 } else if (after(ack, tp->tlp_high_seq)) { 3488 /* ACK advances: there was a loss, so reduce cwnd. Reset 3489 * tlp_high_seq in tcp_init_cwnd_reduction() 3490 */ 3491 tcp_init_cwnd_reduction(sk); 3492 tcp_set_ca_state(sk, TCP_CA_CWR); 3493 tcp_end_cwnd_reduction(sk); 3494 tcp_try_keep_open(sk); 3495 NET_INC_STATS_BH(sock_net(sk), 3496 LINUX_MIB_TCPLOSSPROBERECOVERY); 3497 } else if (!(flag & (FLAG_SND_UNA_ADVANCED | 3498 FLAG_NOT_DUP | FLAG_DATA_SACKED))) { 3499 /* Pure dupack: original and TLP probe arrived; no loss */ 3500 tp->tlp_high_seq = 0; 3501 } 3502 } 3503 3504 static inline void tcp_in_ack_event(struct sock *sk, u32 flags) 3505 { 3506 const struct inet_connection_sock *icsk = inet_csk(sk); 3507 3508 if (icsk->icsk_ca_ops->in_ack_event) 3509 icsk->icsk_ca_ops->in_ack_event(sk, flags); 3510 } 3511 3512 /* This routine deals with incoming acks, but not outgoing ones. */ 3513 static int tcp_ack(struct sock *sk, const struct sk_buff *skb, int flag) 3514 { 3515 struct inet_connection_sock *icsk = inet_csk(sk); 3516 struct tcp_sock *tp = tcp_sk(sk); 3517 struct tcp_sacktag_state sack_state; 3518 u32 prior_snd_una = tp->snd_una; 3519 u32 ack_seq = TCP_SKB_CB(skb)->seq; 3520 u32 ack = TCP_SKB_CB(skb)->ack_seq; 3521 bool is_dupack = false; 3522 u32 prior_fackets; 3523 int prior_packets = tp->packets_out; 3524 const int prior_unsacked = tp->packets_out - tp->sacked_out; 3525 int acked = 0; /* Number of packets newly acked */ 3526 3527 sack_state.first_sackt.v64 = 0; 3528 3529 /* We very likely will need to access write queue head. */ 3530 prefetchw(sk->sk_write_queue.next); 3531 3532 /* If the ack is older than previous acks 3533 * then we can probably ignore it. 3534 */ 3535 if (before(ack, prior_snd_una)) { 3536 /* RFC 5961 5.2 [Blind Data Injection Attack].[Mitigation] */ 3537 if (before(ack, prior_snd_una - tp->max_window)) { 3538 tcp_send_challenge_ack(sk, skb); 3539 return -1; 3540 } 3541 goto old_ack; 3542 } 3543 3544 /* If the ack includes data we haven't sent yet, discard 3545 * this segment (RFC793 Section 3.9). 3546 */ 3547 if (after(ack, tp->snd_nxt)) 3548 goto invalid_ack; 3549 3550 if (icsk->icsk_pending == ICSK_TIME_EARLY_RETRANS || 3551 icsk->icsk_pending == ICSK_TIME_LOSS_PROBE) 3552 tcp_rearm_rto(sk); 3553 3554 if (after(ack, prior_snd_una)) { 3555 flag |= FLAG_SND_UNA_ADVANCED; 3556 icsk->icsk_retransmits = 0; 3557 } 3558 3559 prior_fackets = tp->fackets_out; 3560 3561 /* ts_recent update must be made after we are sure that the packet 3562 * is in window. 3563 */ 3564 if (flag & FLAG_UPDATE_TS_RECENT) 3565 tcp_replace_ts_recent(tp, TCP_SKB_CB(skb)->seq); 3566 3567 if (!(flag & FLAG_SLOWPATH) && after(ack, prior_snd_una)) { 3568 /* Window is constant, pure forward advance. 3569 * No more checks are required. 3570 * Note, we use the fact that SND.UNA>=SND.WL2. 3571 */ 3572 tcp_update_wl(tp, ack_seq); 3573 tcp_snd_una_update(tp, ack); 3574 flag |= FLAG_WIN_UPDATE; 3575 3576 tcp_in_ack_event(sk, CA_ACK_WIN_UPDATE); 3577 3578 NET_INC_STATS_BH(sock_net(sk), LINUX_MIB_TCPHPACKS); 3579 } else { 3580 u32 ack_ev_flags = CA_ACK_SLOWPATH; 3581 3582 if (ack_seq != TCP_SKB_CB(skb)->end_seq) 3583 flag |= FLAG_DATA; 3584 else 3585 NET_INC_STATS_BH(sock_net(sk), LINUX_MIB_TCPPUREACKS); 3586 3587 flag |= tcp_ack_update_window(sk, skb, ack, ack_seq); 3588 3589 if (TCP_SKB_CB(skb)->sacked) 3590 flag |= tcp_sacktag_write_queue(sk, skb, prior_snd_una, 3591 &sack_state); 3592 3593 if (tcp_ecn_rcv_ecn_echo(tp, tcp_hdr(skb))) { 3594 flag |= FLAG_ECE; 3595 ack_ev_flags |= CA_ACK_ECE; 3596 } 3597 3598 if (flag & FLAG_WIN_UPDATE) 3599 ack_ev_flags |= CA_ACK_WIN_UPDATE; 3600 3601 tcp_in_ack_event(sk, ack_ev_flags); 3602 } 3603 3604 /* We passed data and got it acked, remove any soft error 3605 * log. Something worked... 3606 */ 3607 sk->sk_err_soft = 0; 3608 icsk->icsk_probes_out = 0; 3609 tp->rcv_tstamp = tcp_time_stamp; 3610 if (!prior_packets) 3611 goto no_queue; 3612 3613 /* See if we can take anything off of the retransmit queue. */ 3614 acked = tp->packets_out; 3615 flag |= tcp_clean_rtx_queue(sk, prior_fackets, prior_snd_una, 3616 &sack_state); 3617 acked -= tp->packets_out; 3618 3619 if (tcp_ack_is_dubious(sk, flag)) { 3620 is_dupack = !(flag & (FLAG_SND_UNA_ADVANCED | FLAG_NOT_DUP)); 3621 tcp_fastretrans_alert(sk, acked, prior_unsacked, 3622 is_dupack, flag); 3623 } 3624 if (tp->tlp_high_seq) 3625 tcp_process_tlp_ack(sk, ack, flag); 3626 3627 /* Advance cwnd if state allows */ 3628 if (tcp_may_raise_cwnd(sk, flag)) 3629 tcp_cong_avoid(sk, ack, acked); 3630 3631 if ((flag & FLAG_FORWARD_PROGRESS) || !(flag & FLAG_NOT_DUP)) { 3632 struct dst_entry *dst = __sk_dst_get(sk); 3633 if (dst) 3634 dst_confirm(dst); 3635 } 3636 3637 if (icsk->icsk_pending == ICSK_TIME_RETRANS) 3638 tcp_schedule_loss_probe(sk); 3639 tcp_update_pacing_rate(sk); 3640 return 1; 3641 3642 no_queue: 3643 /* If data was DSACKed, see if we can undo a cwnd reduction. */ 3644 if (flag & FLAG_DSACKING_ACK) 3645 tcp_fastretrans_alert(sk, acked, prior_unsacked, 3646 is_dupack, flag); 3647 /* If this ack opens up a zero window, clear backoff. It was 3648 * being used to time the probes, and is probably far higher than 3649 * it needs to be for normal retransmission. 3650 */ 3651 if (tcp_send_head(sk)) 3652 tcp_ack_probe(sk); 3653 3654 if (tp->tlp_high_seq) 3655 tcp_process_tlp_ack(sk, ack, flag); 3656 return 1; 3657 3658 invalid_ack: 3659 SOCK_DEBUG(sk, "Ack %u after %u:%u\n", ack, tp->snd_una, tp->snd_nxt); 3660 return -1; 3661 3662 old_ack: 3663 /* If data was SACKed, tag it and see if we should send more data. 3664 * If data was DSACKed, see if we can undo a cwnd reduction. 3665 */ 3666 if (TCP_SKB_CB(skb)->sacked) { 3667 flag |= tcp_sacktag_write_queue(sk, skb, prior_snd_una, 3668 &sack_state); 3669 tcp_fastretrans_alert(sk, acked, prior_unsacked, 3670 is_dupack, flag); 3671 } 3672 3673 SOCK_DEBUG(sk, "Ack %u before %u:%u\n", ack, tp->snd_una, tp->snd_nxt); 3674 return 0; 3675 } 3676 3677 static void tcp_parse_fastopen_option(int len, const unsigned char *cookie, 3678 bool syn, struct tcp_fastopen_cookie *foc, 3679 bool exp_opt) 3680 { 3681 /* Valid only in SYN or SYN-ACK with an even length. */ 3682 if (!foc || !syn || len < 0 || (len & 1)) 3683 return; 3684 3685 if (len >= TCP_FASTOPEN_COOKIE_MIN && 3686 len <= TCP_FASTOPEN_COOKIE_MAX) 3687 memcpy(foc->val, cookie, len); 3688 else if (len != 0) 3689 len = -1; 3690 foc->len = len; 3691 foc->exp = exp_opt; 3692 } 3693 3694 /* Look for tcp options. Normally only called on SYN and SYNACK packets. 3695 * But, this can also be called on packets in the established flow when 3696 * the fast version below fails. 3697 */ 3698 void tcp_parse_options(const struct sk_buff *skb, 3699 struct tcp_options_received *opt_rx, int estab, 3700 struct tcp_fastopen_cookie *foc) 3701 { 3702 const unsigned char *ptr; 3703 const struct tcphdr *th = tcp_hdr(skb); 3704 int length = (th->doff * 4) - sizeof(struct tcphdr); 3705 3706 ptr = (const unsigned char *)(th + 1); 3707 opt_rx->saw_tstamp = 0; 3708 3709 while (length > 0) { 3710 int opcode = *ptr++; 3711 int opsize; 3712 3713 switch (opcode) { 3714 case TCPOPT_EOL: 3715 return; 3716 case TCPOPT_NOP: /* Ref: RFC 793 section 3.1 */ 3717 length--; 3718 continue; 3719 default: 3720 opsize = *ptr++; 3721 if (opsize < 2) /* "silly options" */ 3722 return; 3723 if (opsize > length) 3724 return; /* don't parse partial options */ 3725 switch (opcode) { 3726 case TCPOPT_MSS: 3727 if (opsize == TCPOLEN_MSS && th->syn && !estab) { 3728 u16 in_mss = get_unaligned_be16(ptr); 3729 if (in_mss) { 3730 if (opt_rx->user_mss && 3731 opt_rx->user_mss < in_mss) 3732 in_mss = opt_rx->user_mss; 3733 opt_rx->mss_clamp = in_mss; 3734 } 3735 } 3736 break; 3737 case TCPOPT_WINDOW: 3738 if (opsize == TCPOLEN_WINDOW && th->syn && 3739 !estab && sysctl_tcp_window_scaling) { 3740 __u8 snd_wscale = *(__u8 *)ptr; 3741 opt_rx->wscale_ok = 1; 3742 if (snd_wscale > 14) { 3743 net_info_ratelimited("%s: Illegal window scaling value %d >14 received\n", 3744 __func__, 3745 snd_wscale); 3746 snd_wscale = 14; 3747 } 3748 opt_rx->snd_wscale = snd_wscale; 3749 } 3750 break; 3751 case TCPOPT_TIMESTAMP: 3752 if ((opsize == TCPOLEN_TIMESTAMP) && 3753 ((estab && opt_rx->tstamp_ok) || 3754 (!estab && sysctl_tcp_timestamps))) { 3755 opt_rx->saw_tstamp = 1; 3756 opt_rx->rcv_tsval = get_unaligned_be32(ptr); 3757 opt_rx->rcv_tsecr = get_unaligned_be32(ptr + 4); 3758 } 3759 break; 3760 case TCPOPT_SACK_PERM: 3761 if (opsize == TCPOLEN_SACK_PERM && th->syn && 3762 !estab && sysctl_tcp_sack) { 3763 opt_rx->sack_ok = TCP_SACK_SEEN; 3764 tcp_sack_reset(opt_rx); 3765 } 3766 break; 3767 3768 case TCPOPT_SACK: 3769 if ((opsize >= (TCPOLEN_SACK_BASE + TCPOLEN_SACK_PERBLOCK)) && 3770 !((opsize - TCPOLEN_SACK_BASE) % TCPOLEN_SACK_PERBLOCK) && 3771 opt_rx->sack_ok) { 3772 TCP_SKB_CB(skb)->sacked = (ptr - 2) - (unsigned char *)th; 3773 } 3774 break; 3775 #ifdef CONFIG_TCP_MD5SIG 3776 case TCPOPT_MD5SIG: 3777 /* 3778 * The MD5 Hash has already been 3779 * checked (see tcp_v{4,6}_do_rcv()). 3780 */ 3781 break; 3782 #endif 3783 case TCPOPT_FASTOPEN: 3784 tcp_parse_fastopen_option( 3785 opsize - TCPOLEN_FASTOPEN_BASE, 3786 ptr, th->syn, foc, false); 3787 break; 3788 3789 case TCPOPT_EXP: 3790 /* Fast Open option shares code 254 using a 3791 * 16 bits magic number. 3792 */ 3793 if (opsize >= TCPOLEN_EXP_FASTOPEN_BASE && 3794 get_unaligned_be16(ptr) == 3795 TCPOPT_FASTOPEN_MAGIC) 3796 tcp_parse_fastopen_option(opsize - 3797 TCPOLEN_EXP_FASTOPEN_BASE, 3798 ptr + 2, th->syn, foc, true); 3799 break; 3800 3801 } 3802 ptr += opsize-2; 3803 length -= opsize; 3804 } 3805 } 3806 } 3807 EXPORT_SYMBOL(tcp_parse_options); 3808 3809 static bool tcp_parse_aligned_timestamp(struct tcp_sock *tp, const struct tcphdr *th) 3810 { 3811 const __be32 *ptr = (const __be32 *)(th + 1); 3812 3813 if (*ptr == htonl((TCPOPT_NOP << 24) | (TCPOPT_NOP << 16) 3814 | (TCPOPT_TIMESTAMP << 8) | TCPOLEN_TIMESTAMP)) { 3815 tp->rx_opt.saw_tstamp = 1; 3816 ++ptr; 3817 tp->rx_opt.rcv_tsval = ntohl(*ptr); 3818 ++ptr; 3819 if (*ptr) 3820 tp->rx_opt.rcv_tsecr = ntohl(*ptr) - tp->tsoffset; 3821 else 3822 tp->rx_opt.rcv_tsecr = 0; 3823 return true; 3824 } 3825 return false; 3826 } 3827 3828 /* Fast parse options. This hopes to only see timestamps. 3829 * If it is wrong it falls back on tcp_parse_options(). 3830 */ 3831 static bool tcp_fast_parse_options(const struct sk_buff *skb, 3832 const struct tcphdr *th, struct tcp_sock *tp) 3833 { 3834 /* In the spirit of fast parsing, compare doff directly to constant 3835 * values. Because equality is used, short doff can be ignored here. 3836 */ 3837 if (th->doff == (sizeof(*th) / 4)) { 3838 tp->rx_opt.saw_tstamp = 0; 3839 return false; 3840 } else if (tp->rx_opt.tstamp_ok && 3841 th->doff == ((sizeof(*th) + TCPOLEN_TSTAMP_ALIGNED) / 4)) { 3842 if (tcp_parse_aligned_timestamp(tp, th)) 3843 return true; 3844 } 3845 3846 tcp_parse_options(skb, &tp->rx_opt, 1, NULL); 3847 if (tp->rx_opt.saw_tstamp && tp->rx_opt.rcv_tsecr) 3848 tp->rx_opt.rcv_tsecr -= tp->tsoffset; 3849 3850 return true; 3851 } 3852 3853 #ifdef CONFIG_TCP_MD5SIG 3854 /* 3855 * Parse MD5 Signature option 3856 */ 3857 const u8 *tcp_parse_md5sig_option(const struct tcphdr *th) 3858 { 3859 int length = (th->doff << 2) - sizeof(*th); 3860 const u8 *ptr = (const u8 *)(th + 1); 3861 3862 /* If the TCP option is too short, we can short cut */ 3863 if (length < TCPOLEN_MD5SIG) 3864 return NULL; 3865 3866 while (length > 0) { 3867 int opcode = *ptr++; 3868 int opsize; 3869 3870 switch (opcode) { 3871 case TCPOPT_EOL: 3872 return NULL; 3873 case TCPOPT_NOP: 3874 length--; 3875 continue; 3876 default: 3877 opsize = *ptr++; 3878 if (opsize < 2 || opsize > length) 3879 return NULL; 3880 if (opcode == TCPOPT_MD5SIG) 3881 return opsize == TCPOLEN_MD5SIG ? ptr : NULL; 3882 } 3883 ptr += opsize - 2; 3884 length -= opsize; 3885 } 3886 return NULL; 3887 } 3888 EXPORT_SYMBOL(tcp_parse_md5sig_option); 3889 #endif 3890 3891 /* Sorry, PAWS as specified is broken wrt. pure-ACKs -DaveM 3892 * 3893 * It is not fatal. If this ACK does _not_ change critical state (seqs, window) 3894 * it can pass through stack. So, the following predicate verifies that 3895 * this segment is not used for anything but congestion avoidance or 3896 * fast retransmit. Moreover, we even are able to eliminate most of such 3897 * second order effects, if we apply some small "replay" window (~RTO) 3898 * to timestamp space. 3899 * 3900 * All these measures still do not guarantee that we reject wrapped ACKs 3901 * on networks with high bandwidth, when sequence space is recycled fastly, 3902 * but it guarantees that such events will be very rare and do not affect 3903 * connection seriously. This doesn't look nice, but alas, PAWS is really 3904 * buggy extension. 3905 * 3906 * [ Later note. Even worse! It is buggy for segments _with_ data. RFC 3907 * states that events when retransmit arrives after original data are rare. 3908 * It is a blatant lie. VJ forgot about fast retransmit! 8)8) It is 3909 * the biggest problem on large power networks even with minor reordering. 3910 * OK, let's give it small replay window. If peer clock is even 1hz, it is safe 3911 * up to bandwidth of 18Gigabit/sec. 8) ] 3912 */ 3913 3914 static int tcp_disordered_ack(const struct sock *sk, const struct sk_buff *skb) 3915 { 3916 const struct tcp_sock *tp = tcp_sk(sk); 3917 const struct tcphdr *th = tcp_hdr(skb); 3918 u32 seq = TCP_SKB_CB(skb)->seq; 3919 u32 ack = TCP_SKB_CB(skb)->ack_seq; 3920 3921 return (/* 1. Pure ACK with correct sequence number. */ 3922 (th->ack && seq == TCP_SKB_CB(skb)->end_seq && seq == tp->rcv_nxt) && 3923 3924 /* 2. ... and duplicate ACK. */ 3925 ack == tp->snd_una && 3926 3927 /* 3. ... and does not update window. */ 3928 !tcp_may_update_window(tp, ack, seq, ntohs(th->window) << tp->rx_opt.snd_wscale) && 3929 3930 /* 4. ... and sits in replay window. */ 3931 (s32)(tp->rx_opt.ts_recent - tp->rx_opt.rcv_tsval) <= (inet_csk(sk)->icsk_rto * 1024) / HZ); 3932 } 3933 3934 static inline bool tcp_paws_discard(const struct sock *sk, 3935 const struct sk_buff *skb) 3936 { 3937 const struct tcp_sock *tp = tcp_sk(sk); 3938 3939 return !tcp_paws_check(&tp->rx_opt, TCP_PAWS_WINDOW) && 3940 !tcp_disordered_ack(sk, skb); 3941 } 3942 3943 /* Check segment sequence number for validity. 3944 * 3945 * Segment controls are considered valid, if the segment 3946 * fits to the window after truncation to the window. Acceptability 3947 * of data (and SYN, FIN, of course) is checked separately. 3948 * See tcp_data_queue(), for example. 3949 * 3950 * Also, controls (RST is main one) are accepted using RCV.WUP instead 3951 * of RCV.NXT. Peer still did not advance his SND.UNA when we 3952 * delayed ACK, so that hisSND.UNA<=ourRCV.WUP. 3953 * (borrowed from freebsd) 3954 */ 3955 3956 static inline bool tcp_sequence(const struct tcp_sock *tp, u32 seq, u32 end_seq) 3957 { 3958 return !before(end_seq, tp->rcv_wup) && 3959 !after(seq, tp->rcv_nxt + tcp_receive_window(tp)); 3960 } 3961 3962 /* When we get a reset we do this. */ 3963 void tcp_reset(struct sock *sk) 3964 { 3965 /* We want the right error as BSD sees it (and indeed as we do). */ 3966 switch (sk->sk_state) { 3967 case TCP_SYN_SENT: 3968 sk->sk_err = ECONNREFUSED; 3969 break; 3970 case TCP_CLOSE_WAIT: 3971 sk->sk_err = EPIPE; 3972 break; 3973 case TCP_CLOSE: 3974 return; 3975 default: 3976 sk->sk_err = ECONNRESET; 3977 } 3978 /* This barrier is coupled with smp_rmb() in tcp_poll() */ 3979 smp_wmb(); 3980 3981 if (!sock_flag(sk, SOCK_DEAD)) 3982 sk->sk_error_report(sk); 3983 3984 tcp_done(sk); 3985 } 3986 3987 /* 3988 * Process the FIN bit. This now behaves as it is supposed to work 3989 * and the FIN takes effect when it is validly part of sequence 3990 * space. Not before when we get holes. 3991 * 3992 * If we are ESTABLISHED, a received fin moves us to CLOSE-WAIT 3993 * (and thence onto LAST-ACK and finally, CLOSE, we never enter 3994 * TIME-WAIT) 3995 * 3996 * If we are in FINWAIT-1, a received FIN indicates simultaneous 3997 * close and we go into CLOSING (and later onto TIME-WAIT) 3998 * 3999 * If we are in FINWAIT-2, a received FIN moves us to TIME-WAIT. 4000 */ 4001 static void tcp_fin(struct sock *sk) 4002 { 4003 struct tcp_sock *tp = tcp_sk(sk); 4004 4005 inet_csk_schedule_ack(sk); 4006 4007 sk->sk_shutdown |= RCV_SHUTDOWN; 4008 sock_set_flag(sk, SOCK_DONE); 4009 4010 switch (sk->sk_state) { 4011 case TCP_SYN_RECV: 4012 case TCP_ESTABLISHED: 4013 /* Move to CLOSE_WAIT */ 4014 tcp_set_state(sk, TCP_CLOSE_WAIT); 4015 inet_csk(sk)->icsk_ack.pingpong = 1; 4016 break; 4017 4018 case TCP_CLOSE_WAIT: 4019 case TCP_CLOSING: 4020 /* Received a retransmission of the FIN, do 4021 * nothing. 4022 */ 4023 break; 4024 case TCP_LAST_ACK: 4025 /* RFC793: Remain in the LAST-ACK state. */ 4026 break; 4027 4028 case TCP_FIN_WAIT1: 4029 /* This case occurs when a simultaneous close 4030 * happens, we must ack the received FIN and 4031 * enter the CLOSING state. 4032 */ 4033 tcp_send_ack(sk); 4034 tcp_set_state(sk, TCP_CLOSING); 4035 break; 4036 case TCP_FIN_WAIT2: 4037 /* Received a FIN -- send ACK and enter TIME_WAIT. */ 4038 tcp_send_ack(sk); 4039 tcp_time_wait(sk, TCP_TIME_WAIT, 0); 4040 break; 4041 default: 4042 /* Only TCP_LISTEN and TCP_CLOSE are left, in these 4043 * cases we should never reach this piece of code. 4044 */ 4045 pr_err("%s: Impossible, sk->sk_state=%d\n", 4046 __func__, sk->sk_state); 4047 break; 4048 } 4049 4050 /* It _is_ possible, that we have something out-of-order _after_ FIN. 4051 * Probably, we should reset in this case. For now drop them. 4052 */ 4053 __skb_queue_purge(&tp->out_of_order_queue); 4054 if (tcp_is_sack(tp)) 4055 tcp_sack_reset(&tp->rx_opt); 4056 sk_mem_reclaim(sk); 4057 4058 if (!sock_flag(sk, SOCK_DEAD)) { 4059 sk->sk_state_change(sk); 4060 4061 /* Do not send POLL_HUP for half duplex close. */ 4062 if (sk->sk_shutdown == SHUTDOWN_MASK || 4063 sk->sk_state == TCP_CLOSE) 4064 sk_wake_async(sk, SOCK_WAKE_WAITD, POLL_HUP); 4065 else 4066 sk_wake_async(sk, SOCK_WAKE_WAITD, POLL_IN); 4067 } 4068 } 4069 4070 static inline bool tcp_sack_extend(struct tcp_sack_block *sp, u32 seq, 4071 u32 end_seq) 4072 { 4073 if (!after(seq, sp->end_seq) && !after(sp->start_seq, end_seq)) { 4074 if (before(seq, sp->start_seq)) 4075 sp->start_seq = seq; 4076 if (after(end_seq, sp->end_seq)) 4077 sp->end_seq = end_seq; 4078 return true; 4079 } 4080 return false; 4081 } 4082 4083 static void tcp_dsack_set(struct sock *sk, u32 seq, u32 end_seq) 4084 { 4085 struct tcp_sock *tp = tcp_sk(sk); 4086 4087 if (tcp_is_sack(tp) && sysctl_tcp_dsack) { 4088 int mib_idx; 4089 4090 if (before(seq, tp->rcv_nxt)) 4091 mib_idx = LINUX_MIB_TCPDSACKOLDSENT; 4092 else 4093 mib_idx = LINUX_MIB_TCPDSACKOFOSENT; 4094 4095 NET_INC_STATS_BH(sock_net(sk), mib_idx); 4096 4097 tp->rx_opt.dsack = 1; 4098 tp->duplicate_sack[0].start_seq = seq; 4099 tp->duplicate_sack[0].end_seq = end_seq; 4100 } 4101 } 4102 4103 static void tcp_dsack_extend(struct sock *sk, u32 seq, u32 end_seq) 4104 { 4105 struct tcp_sock *tp = tcp_sk(sk); 4106 4107 if (!tp->rx_opt.dsack) 4108 tcp_dsack_set(sk, seq, end_seq); 4109 else 4110 tcp_sack_extend(tp->duplicate_sack, seq, end_seq); 4111 } 4112 4113 static void tcp_send_dupack(struct sock *sk, const struct sk_buff *skb) 4114 { 4115 struct tcp_sock *tp = tcp_sk(sk); 4116 4117 if (TCP_SKB_CB(skb)->end_seq != TCP_SKB_CB(skb)->seq && 4118 before(TCP_SKB_CB(skb)->seq, tp->rcv_nxt)) { 4119 NET_INC_STATS_BH(sock_net(sk), LINUX_MIB_DELAYEDACKLOST); 4120 tcp_enter_quickack_mode(sk); 4121 4122 if (tcp_is_sack(tp) && sysctl_tcp_dsack) { 4123 u32 end_seq = TCP_SKB_CB(skb)->end_seq; 4124 4125 if (after(TCP_SKB_CB(skb)->end_seq, tp->rcv_nxt)) 4126 end_seq = tp->rcv_nxt; 4127 tcp_dsack_set(sk, TCP_SKB_CB(skb)->seq, end_seq); 4128 } 4129 } 4130 4131 tcp_send_ack(sk); 4132 } 4133 4134 /* These routines update the SACK block as out-of-order packets arrive or 4135 * in-order packets close up the sequence space. 4136 */ 4137 static void tcp_sack_maybe_coalesce(struct tcp_sock *tp) 4138 { 4139 int this_sack; 4140 struct tcp_sack_block *sp = &tp->selective_acks[0]; 4141 struct tcp_sack_block *swalk = sp + 1; 4142 4143 /* See if the recent change to the first SACK eats into 4144 * or hits the sequence space of other SACK blocks, if so coalesce. 4145 */ 4146 for (this_sack = 1; this_sack < tp->rx_opt.num_sacks;) { 4147 if (tcp_sack_extend(sp, swalk->start_seq, swalk->end_seq)) { 4148 int i; 4149 4150 /* Zap SWALK, by moving every further SACK up by one slot. 4151 * Decrease num_sacks. 4152 */ 4153 tp->rx_opt.num_sacks--; 4154 for (i = this_sack; i < tp->rx_opt.num_sacks; i++) 4155 sp[i] = sp[i + 1]; 4156 continue; 4157 } 4158 this_sack++, swalk++; 4159 } 4160 } 4161 4162 static void tcp_sack_new_ofo_skb(struct sock *sk, u32 seq, u32 end_seq) 4163 { 4164 struct tcp_sock *tp = tcp_sk(sk); 4165 struct tcp_sack_block *sp = &tp->selective_acks[0]; 4166 int cur_sacks = tp->rx_opt.num_sacks; 4167 int this_sack; 4168 4169 if (!cur_sacks) 4170 goto new_sack; 4171 4172 for (this_sack = 0; this_sack < cur_sacks; this_sack++, sp++) { 4173 if (tcp_sack_extend(sp, seq, end_seq)) { 4174 /* Rotate this_sack to the first one. */ 4175 for (; this_sack > 0; this_sack--, sp--) 4176 swap(*sp, *(sp - 1)); 4177 if (cur_sacks > 1) 4178 tcp_sack_maybe_coalesce(tp); 4179 return; 4180 } 4181 } 4182 4183 /* Could not find an adjacent existing SACK, build a new one, 4184 * put it at the front, and shift everyone else down. We 4185 * always know there is at least one SACK present already here. 4186 * 4187 * If the sack array is full, forget about the last one. 4188 */ 4189 if (this_sack >= TCP_NUM_SACKS) { 4190 this_sack--; 4191 tp->rx_opt.num_sacks--; 4192 sp--; 4193 } 4194 for (; this_sack > 0; this_sack--, sp--) 4195 *sp = *(sp - 1); 4196 4197 new_sack: 4198 /* Build the new head SACK, and we're done. */ 4199 sp->start_seq = seq; 4200 sp->end_seq = end_seq; 4201 tp->rx_opt.num_sacks++; 4202 } 4203 4204 /* RCV.NXT advances, some SACKs should be eaten. */ 4205 4206 static void tcp_sack_remove(struct tcp_sock *tp) 4207 { 4208 struct tcp_sack_block *sp = &tp->selective_acks[0]; 4209 int num_sacks = tp->rx_opt.num_sacks; 4210 int this_sack; 4211 4212 /* Empty ofo queue, hence, all the SACKs are eaten. Clear. */ 4213 if (skb_queue_empty(&tp->out_of_order_queue)) { 4214 tp->rx_opt.num_sacks = 0; 4215 return; 4216 } 4217 4218 for (this_sack = 0; this_sack < num_sacks;) { 4219 /* Check if the start of the sack is covered by RCV.NXT. */ 4220 if (!before(tp->rcv_nxt, sp->start_seq)) { 4221 int i; 4222 4223 /* RCV.NXT must cover all the block! */ 4224 WARN_ON(before(tp->rcv_nxt, sp->end_seq)); 4225 4226 /* Zap this SACK, by moving forward any other SACKS. */ 4227 for (i = this_sack+1; i < num_sacks; i++) 4228 tp->selective_acks[i-1] = tp->selective_acks[i]; 4229 num_sacks--; 4230 continue; 4231 } 4232 this_sack++; 4233 sp++; 4234 } 4235 tp->rx_opt.num_sacks = num_sacks; 4236 } 4237 4238 /** 4239 * tcp_try_coalesce - try to merge skb to prior one 4240 * @sk: socket 4241 * @to: prior buffer 4242 * @from: buffer to add in queue 4243 * @fragstolen: pointer to boolean 4244 * 4245 * Before queueing skb @from after @to, try to merge them 4246 * to reduce overall memory use and queue lengths, if cost is small. 4247 * Packets in ofo or receive queues can stay a long time. 4248 * Better try to coalesce them right now to avoid future collapses. 4249 * Returns true if caller should free @from instead of queueing it 4250 */ 4251 static bool tcp_try_coalesce(struct sock *sk, 4252 struct sk_buff *to, 4253 struct sk_buff *from, 4254 bool *fragstolen) 4255 { 4256 int delta; 4257 4258 *fragstolen = false; 4259 4260 /* Its possible this segment overlaps with prior segment in queue */ 4261 if (TCP_SKB_CB(from)->seq != TCP_SKB_CB(to)->end_seq) 4262 return false; 4263 4264 if (!skb_try_coalesce(to, from, fragstolen, &delta)) 4265 return false; 4266 4267 atomic_add(delta, &sk->sk_rmem_alloc); 4268 sk_mem_charge(sk, delta); 4269 NET_INC_STATS_BH(sock_net(sk), LINUX_MIB_TCPRCVCOALESCE); 4270 TCP_SKB_CB(to)->end_seq = TCP_SKB_CB(from)->end_seq; 4271 TCP_SKB_CB(to)->ack_seq = TCP_SKB_CB(from)->ack_seq; 4272 TCP_SKB_CB(to)->tcp_flags |= TCP_SKB_CB(from)->tcp_flags; 4273 return true; 4274 } 4275 4276 /* This one checks to see if we can put data from the 4277 * out_of_order queue into the receive_queue. 4278 */ 4279 static void tcp_ofo_queue(struct sock *sk) 4280 { 4281 struct tcp_sock *tp = tcp_sk(sk); 4282 __u32 dsack_high = tp->rcv_nxt; 4283 struct sk_buff *skb, *tail; 4284 bool fragstolen, eaten; 4285 4286 while ((skb = skb_peek(&tp->out_of_order_queue)) != NULL) { 4287 if (after(TCP_SKB_CB(skb)->seq, tp->rcv_nxt)) 4288 break; 4289 4290 if (before(TCP_SKB_CB(skb)->seq, dsack_high)) { 4291 __u32 dsack = dsack_high; 4292 if (before(TCP_SKB_CB(skb)->end_seq, dsack_high)) 4293 dsack_high = TCP_SKB_CB(skb)->end_seq; 4294 tcp_dsack_extend(sk, TCP_SKB_CB(skb)->seq, dsack); 4295 } 4296 4297 __skb_unlink(skb, &tp->out_of_order_queue); 4298 if (!after(TCP_SKB_CB(skb)->end_seq, tp->rcv_nxt)) { 4299 SOCK_DEBUG(sk, "ofo packet was already received\n"); 4300 __kfree_skb(skb); 4301 continue; 4302 } 4303 SOCK_DEBUG(sk, "ofo requeuing : rcv_next %X seq %X - %X\n", 4304 tp->rcv_nxt, TCP_SKB_CB(skb)->seq, 4305 TCP_SKB_CB(skb)->end_seq); 4306 4307 tail = skb_peek_tail(&sk->sk_receive_queue); 4308 eaten = tail && tcp_try_coalesce(sk, tail, skb, &fragstolen); 4309 tcp_rcv_nxt_update(tp, TCP_SKB_CB(skb)->end_seq); 4310 if (!eaten) 4311 __skb_queue_tail(&sk->sk_receive_queue, skb); 4312 if (TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN) 4313 tcp_fin(sk); 4314 if (eaten) 4315 kfree_skb_partial(skb, fragstolen); 4316 } 4317 } 4318 4319 static bool tcp_prune_ofo_queue(struct sock *sk); 4320 static int tcp_prune_queue(struct sock *sk); 4321 4322 static int tcp_try_rmem_schedule(struct sock *sk, struct sk_buff *skb, 4323 unsigned int size) 4324 { 4325 if (atomic_read(&sk->sk_rmem_alloc) > sk->sk_rcvbuf || 4326 !sk_rmem_schedule(sk, skb, size)) { 4327 4328 if (tcp_prune_queue(sk) < 0) 4329 return -1; 4330 4331 if (!sk_rmem_schedule(sk, skb, size)) { 4332 if (!tcp_prune_ofo_queue(sk)) 4333 return -1; 4334 4335 if (!sk_rmem_schedule(sk, skb, size)) 4336 return -1; 4337 } 4338 } 4339 return 0; 4340 } 4341 4342 static void tcp_data_queue_ofo(struct sock *sk, struct sk_buff *skb) 4343 { 4344 struct tcp_sock *tp = tcp_sk(sk); 4345 struct sk_buff *skb1; 4346 u32 seq, end_seq; 4347 4348 tcp_ecn_check_ce(tp, skb); 4349 4350 if (unlikely(tcp_try_rmem_schedule(sk, skb, skb->truesize))) { 4351 NET_INC_STATS_BH(sock_net(sk), LINUX_MIB_TCPOFODROP); 4352 __kfree_skb(skb); 4353 return; 4354 } 4355 4356 /* Disable header prediction. */ 4357 tp->pred_flags = 0; 4358 inet_csk_schedule_ack(sk); 4359 4360 NET_INC_STATS_BH(sock_net(sk), LINUX_MIB_TCPOFOQUEUE); 4361 SOCK_DEBUG(sk, "out of order segment: rcv_next %X seq %X - %X\n", 4362 tp->rcv_nxt, TCP_SKB_CB(skb)->seq, TCP_SKB_CB(skb)->end_seq); 4363 4364 skb1 = skb_peek_tail(&tp->out_of_order_queue); 4365 if (!skb1) { 4366 /* Initial out of order segment, build 1 SACK. */ 4367 if (tcp_is_sack(tp)) { 4368 tp->rx_opt.num_sacks = 1; 4369 tp->selective_acks[0].start_seq = TCP_SKB_CB(skb)->seq; 4370 tp->selective_acks[0].end_seq = 4371 TCP_SKB_CB(skb)->end_seq; 4372 } 4373 __skb_queue_head(&tp->out_of_order_queue, skb); 4374 goto end; 4375 } 4376 4377 seq = TCP_SKB_CB(skb)->seq; 4378 end_seq = TCP_SKB_CB(skb)->end_seq; 4379 4380 if (seq == TCP_SKB_CB(skb1)->end_seq) { 4381 bool fragstolen; 4382 4383 if (!tcp_try_coalesce(sk, skb1, skb, &fragstolen)) { 4384 __skb_queue_after(&tp->out_of_order_queue, skb1, skb); 4385 } else { 4386 tcp_grow_window(sk, skb); 4387 kfree_skb_partial(skb, fragstolen); 4388 skb = NULL; 4389 } 4390 4391 if (!tp->rx_opt.num_sacks || 4392 tp->selective_acks[0].end_seq != seq) 4393 goto add_sack; 4394 4395 /* Common case: data arrive in order after hole. */ 4396 tp->selective_acks[0].end_seq = end_seq; 4397 goto end; 4398 } 4399 4400 /* Find place to insert this segment. */ 4401 while (1) { 4402 if (!after(TCP_SKB_CB(skb1)->seq, seq)) 4403 break; 4404 if (skb_queue_is_first(&tp->out_of_order_queue, skb1)) { 4405 skb1 = NULL; 4406 break; 4407 } 4408 skb1 = skb_queue_prev(&tp->out_of_order_queue, skb1); 4409 } 4410 4411 /* Do skb overlap to previous one? */ 4412 if (skb1 && before(seq, TCP_SKB_CB(skb1)->end_seq)) { 4413 if (!after(end_seq, TCP_SKB_CB(skb1)->end_seq)) { 4414 /* All the bits are present. Drop. */ 4415 NET_INC_STATS_BH(sock_net(sk), LINUX_MIB_TCPOFOMERGE); 4416 __kfree_skb(skb); 4417 skb = NULL; 4418 tcp_dsack_set(sk, seq, end_seq); 4419 goto add_sack; 4420 } 4421 if (after(seq, TCP_SKB_CB(skb1)->seq)) { 4422 /* Partial overlap. */ 4423 tcp_dsack_set(sk, seq, 4424 TCP_SKB_CB(skb1)->end_seq); 4425 } else { 4426 if (skb_queue_is_first(&tp->out_of_order_queue, 4427 skb1)) 4428 skb1 = NULL; 4429 else 4430 skb1 = skb_queue_prev( 4431 &tp->out_of_order_queue, 4432 skb1); 4433 } 4434 } 4435 if (!skb1) 4436 __skb_queue_head(&tp->out_of_order_queue, skb); 4437 else 4438 __skb_queue_after(&tp->out_of_order_queue, skb1, skb); 4439 4440 /* And clean segments covered by new one as whole. */ 4441 while (!skb_queue_is_last(&tp->out_of_order_queue, skb)) { 4442 skb1 = skb_queue_next(&tp->out_of_order_queue, skb); 4443 4444 if (!after(end_seq, TCP_SKB_CB(skb1)->seq)) 4445 break; 4446 if (before(end_seq, TCP_SKB_CB(skb1)->end_seq)) { 4447 tcp_dsack_extend(sk, TCP_SKB_CB(skb1)->seq, 4448 end_seq); 4449 break; 4450 } 4451 __skb_unlink(skb1, &tp->out_of_order_queue); 4452 tcp_dsack_extend(sk, TCP_SKB_CB(skb1)->seq, 4453 TCP_SKB_CB(skb1)->end_seq); 4454 NET_INC_STATS_BH(sock_net(sk), LINUX_MIB_TCPOFOMERGE); 4455 __kfree_skb(skb1); 4456 } 4457 4458 add_sack: 4459 if (tcp_is_sack(tp)) 4460 tcp_sack_new_ofo_skb(sk, seq, end_seq); 4461 end: 4462 if (skb) { 4463 tcp_grow_window(sk, skb); 4464 skb_set_owner_r(skb, sk); 4465 } 4466 } 4467 4468 static int __must_check tcp_queue_rcv(struct sock *sk, struct sk_buff *skb, int hdrlen, 4469 bool *fragstolen) 4470 { 4471 int eaten; 4472 struct sk_buff *tail = skb_peek_tail(&sk->sk_receive_queue); 4473 4474 __skb_pull(skb, hdrlen); 4475 eaten = (tail && 4476 tcp_try_coalesce(sk, tail, skb, fragstolen)) ? 1 : 0; 4477 tcp_rcv_nxt_update(tcp_sk(sk), TCP_SKB_CB(skb)->end_seq); 4478 if (!eaten) { 4479 __skb_queue_tail(&sk->sk_receive_queue, skb); 4480 skb_set_owner_r(skb, sk); 4481 } 4482 return eaten; 4483 } 4484 4485 int tcp_send_rcvq(struct sock *sk, struct msghdr *msg, size_t size) 4486 { 4487 struct sk_buff *skb; 4488 int err = -ENOMEM; 4489 int data_len = 0; 4490 bool fragstolen; 4491 4492 if (size == 0) 4493 return 0; 4494 4495 if (size > PAGE_SIZE) { 4496 int npages = min_t(size_t, size >> PAGE_SHIFT, MAX_SKB_FRAGS); 4497 4498 data_len = npages << PAGE_SHIFT; 4499 size = data_len + (size & ~PAGE_MASK); 4500 } 4501 skb = alloc_skb_with_frags(size - data_len, data_len, 4502 PAGE_ALLOC_COSTLY_ORDER, 4503 &err, sk->sk_allocation); 4504 if (!skb) 4505 goto err; 4506 4507 skb_put(skb, size - data_len); 4508 skb->data_len = data_len; 4509 skb->len = size; 4510 4511 if (tcp_try_rmem_schedule(sk, skb, skb->truesize)) 4512 goto err_free; 4513 4514 err = skb_copy_datagram_from_iter(skb, 0, &msg->msg_iter, size); 4515 if (err) 4516 goto err_free; 4517 4518 TCP_SKB_CB(skb)->seq = tcp_sk(sk)->rcv_nxt; 4519 TCP_SKB_CB(skb)->end_seq = TCP_SKB_CB(skb)->seq + size; 4520 TCP_SKB_CB(skb)->ack_seq = tcp_sk(sk)->snd_una - 1; 4521 4522 if (tcp_queue_rcv(sk, skb, 0, &fragstolen)) { 4523 WARN_ON_ONCE(fragstolen); /* should not happen */ 4524 __kfree_skb(skb); 4525 } 4526 return size; 4527 4528 err_free: 4529 kfree_skb(skb); 4530 err: 4531 return err; 4532 4533 } 4534 4535 static void tcp_data_queue(struct sock *sk, struct sk_buff *skb) 4536 { 4537 struct tcp_sock *tp = tcp_sk(sk); 4538 int eaten = -1; 4539 bool fragstolen = false; 4540 4541 if (TCP_SKB_CB(skb)->seq == TCP_SKB_CB(skb)->end_seq) 4542 goto drop; 4543 4544 skb_dst_drop(skb); 4545 __skb_pull(skb, tcp_hdr(skb)->doff * 4); 4546 4547 tcp_ecn_accept_cwr(tp, skb); 4548 4549 tp->rx_opt.dsack = 0; 4550 4551 /* Queue data for delivery to the user. 4552 * Packets in sequence go to the receive queue. 4553 * Out of sequence packets to the out_of_order_queue. 4554 */ 4555 if (TCP_SKB_CB(skb)->seq == tp->rcv_nxt) { 4556 if (tcp_receive_window(tp) == 0) 4557 goto out_of_window; 4558 4559 /* Ok. In sequence. In window. */ 4560 if (tp->ucopy.task == current && 4561 tp->copied_seq == tp->rcv_nxt && tp->ucopy.len && 4562 sock_owned_by_user(sk) && !tp->urg_data) { 4563 int chunk = min_t(unsigned int, skb->len, 4564 tp->ucopy.len); 4565 4566 __set_current_state(TASK_RUNNING); 4567 4568 local_bh_enable(); 4569 if (!skb_copy_datagram_msg(skb, 0, tp->ucopy.msg, chunk)) { 4570 tp->ucopy.len -= chunk; 4571 tp->copied_seq += chunk; 4572 eaten = (chunk == skb->len); 4573 tcp_rcv_space_adjust(sk); 4574 } 4575 local_bh_disable(); 4576 } 4577 4578 if (eaten <= 0) { 4579 queue_and_out: 4580 if (eaten < 0) { 4581 if (skb_queue_len(&sk->sk_receive_queue) == 0) 4582 sk_forced_mem_schedule(sk, skb->truesize); 4583 else if (tcp_try_rmem_schedule(sk, skb, skb->truesize)) 4584 goto drop; 4585 } 4586 eaten = tcp_queue_rcv(sk, skb, 0, &fragstolen); 4587 } 4588 tcp_rcv_nxt_update(tp, TCP_SKB_CB(skb)->end_seq); 4589 if (skb->len) 4590 tcp_event_data_recv(sk, skb); 4591 if (TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN) 4592 tcp_fin(sk); 4593 4594 if (!skb_queue_empty(&tp->out_of_order_queue)) { 4595 tcp_ofo_queue(sk); 4596 4597 /* RFC2581. 4.2. SHOULD send immediate ACK, when 4598 * gap in queue is filled. 4599 */ 4600 if (skb_queue_empty(&tp->out_of_order_queue)) 4601 inet_csk(sk)->icsk_ack.pingpong = 0; 4602 } 4603 4604 if (tp->rx_opt.num_sacks) 4605 tcp_sack_remove(tp); 4606 4607 tcp_fast_path_check(sk); 4608 4609 if (eaten > 0) 4610 kfree_skb_partial(skb, fragstolen); 4611 if (!sock_flag(sk, SOCK_DEAD)) 4612 sk->sk_data_ready(sk); 4613 return; 4614 } 4615 4616 if (!after(TCP_SKB_CB(skb)->end_seq, tp->rcv_nxt)) { 4617 /* A retransmit, 2nd most common case. Force an immediate ack. */ 4618 NET_INC_STATS_BH(sock_net(sk), LINUX_MIB_DELAYEDACKLOST); 4619 tcp_dsack_set(sk, TCP_SKB_CB(skb)->seq, TCP_SKB_CB(skb)->end_seq); 4620 4621 out_of_window: 4622 tcp_enter_quickack_mode(sk); 4623 inet_csk_schedule_ack(sk); 4624 drop: 4625 __kfree_skb(skb); 4626 return; 4627 } 4628 4629 /* Out of window. F.e. zero window probe. */ 4630 if (!before(TCP_SKB_CB(skb)->seq, tp->rcv_nxt + tcp_receive_window(tp))) 4631 goto out_of_window; 4632 4633 tcp_enter_quickack_mode(sk); 4634 4635 if (before(TCP_SKB_CB(skb)->seq, tp->rcv_nxt)) { 4636 /* Partial packet, seq < rcv_next < end_seq */ 4637 SOCK_DEBUG(sk, "partial packet: rcv_next %X seq %X - %X\n", 4638 tp->rcv_nxt, TCP_SKB_CB(skb)->seq, 4639 TCP_SKB_CB(skb)->end_seq); 4640 4641 tcp_dsack_set(sk, TCP_SKB_CB(skb)->seq, tp->rcv_nxt); 4642 4643 /* If window is closed, drop tail of packet. But after 4644 * remembering D-SACK for its head made in previous line. 4645 */ 4646 if (!tcp_receive_window(tp)) 4647 goto out_of_window; 4648 goto queue_and_out; 4649 } 4650 4651 tcp_data_queue_ofo(sk, skb); 4652 } 4653 4654 static struct sk_buff *tcp_collapse_one(struct sock *sk, struct sk_buff *skb, 4655 struct sk_buff_head *list) 4656 { 4657 struct sk_buff *next = NULL; 4658 4659 if (!skb_queue_is_last(list, skb)) 4660 next = skb_queue_next(list, skb); 4661 4662 __skb_unlink(skb, list); 4663 __kfree_skb(skb); 4664 NET_INC_STATS_BH(sock_net(sk), LINUX_MIB_TCPRCVCOLLAPSED); 4665 4666 return next; 4667 } 4668 4669 /* Collapse contiguous sequence of skbs head..tail with 4670 * sequence numbers start..end. 4671 * 4672 * If tail is NULL, this means until the end of the list. 4673 * 4674 * Segments with FIN/SYN are not collapsed (only because this 4675 * simplifies code) 4676 */ 4677 static void 4678 tcp_collapse(struct sock *sk, struct sk_buff_head *list, 4679 struct sk_buff *head, struct sk_buff *tail, 4680 u32 start, u32 end) 4681 { 4682 struct sk_buff *skb, *n; 4683 bool end_of_skbs; 4684 4685 /* First, check that queue is collapsible and find 4686 * the point where collapsing can be useful. */ 4687 skb = head; 4688 restart: 4689 end_of_skbs = true; 4690 skb_queue_walk_from_safe(list, skb, n) { 4691 if (skb == tail) 4692 break; 4693 /* No new bits? It is possible on ofo queue. */ 4694 if (!before(start, TCP_SKB_CB(skb)->end_seq)) { 4695 skb = tcp_collapse_one(sk, skb, list); 4696 if (!skb) 4697 break; 4698 goto restart; 4699 } 4700 4701 /* The first skb to collapse is: 4702 * - not SYN/FIN and 4703 * - bloated or contains data before "start" or 4704 * overlaps to the next one. 4705 */ 4706 if (!(TCP_SKB_CB(skb)->tcp_flags & (TCPHDR_SYN | TCPHDR_FIN)) && 4707 (tcp_win_from_space(skb->truesize) > skb->len || 4708 before(TCP_SKB_CB(skb)->seq, start))) { 4709 end_of_skbs = false; 4710 break; 4711 } 4712 4713 if (!skb_queue_is_last(list, skb)) { 4714 struct sk_buff *next = skb_queue_next(list, skb); 4715 if (next != tail && 4716 TCP_SKB_CB(skb)->end_seq != TCP_SKB_CB(next)->seq) { 4717 end_of_skbs = false; 4718 break; 4719 } 4720 } 4721 4722 /* Decided to skip this, advance start seq. */ 4723 start = TCP_SKB_CB(skb)->end_seq; 4724 } 4725 if (end_of_skbs || 4726 (TCP_SKB_CB(skb)->tcp_flags & (TCPHDR_SYN | TCPHDR_FIN))) 4727 return; 4728 4729 while (before(start, end)) { 4730 int copy = min_t(int, SKB_MAX_ORDER(0, 0), end - start); 4731 struct sk_buff *nskb; 4732 4733 nskb = alloc_skb(copy, GFP_ATOMIC); 4734 if (!nskb) 4735 return; 4736 4737 memcpy(nskb->cb, skb->cb, sizeof(skb->cb)); 4738 TCP_SKB_CB(nskb)->seq = TCP_SKB_CB(nskb)->end_seq = start; 4739 __skb_queue_before(list, skb, nskb); 4740 skb_set_owner_r(nskb, sk); 4741 4742 /* Copy data, releasing collapsed skbs. */ 4743 while (copy > 0) { 4744 int offset = start - TCP_SKB_CB(skb)->seq; 4745 int size = TCP_SKB_CB(skb)->end_seq - start; 4746 4747 BUG_ON(offset < 0); 4748 if (size > 0) { 4749 size = min(copy, size); 4750 if (skb_copy_bits(skb, offset, skb_put(nskb, size), size)) 4751 BUG(); 4752 TCP_SKB_CB(nskb)->end_seq += size; 4753 copy -= size; 4754 start += size; 4755 } 4756 if (!before(start, TCP_SKB_CB(skb)->end_seq)) { 4757 skb = tcp_collapse_one(sk, skb, list); 4758 if (!skb || 4759 skb == tail || 4760 (TCP_SKB_CB(skb)->tcp_flags & (TCPHDR_SYN | TCPHDR_FIN))) 4761 return; 4762 } 4763 } 4764 } 4765 } 4766 4767 /* Collapse ofo queue. Algorithm: select contiguous sequence of skbs 4768 * and tcp_collapse() them until all the queue is collapsed. 4769 */ 4770 static void tcp_collapse_ofo_queue(struct sock *sk) 4771 { 4772 struct tcp_sock *tp = tcp_sk(sk); 4773 struct sk_buff *skb = skb_peek(&tp->out_of_order_queue); 4774 struct sk_buff *head; 4775 u32 start, end; 4776 4777 if (!skb) 4778 return; 4779 4780 start = TCP_SKB_CB(skb)->seq; 4781 end = TCP_SKB_CB(skb)->end_seq; 4782 head = skb; 4783 4784 for (;;) { 4785 struct sk_buff *next = NULL; 4786 4787 if (!skb_queue_is_last(&tp->out_of_order_queue, skb)) 4788 next = skb_queue_next(&tp->out_of_order_queue, skb); 4789 skb = next; 4790 4791 /* Segment is terminated when we see gap or when 4792 * we are at the end of all the queue. */ 4793 if (!skb || 4794 after(TCP_SKB_CB(skb)->seq, end) || 4795 before(TCP_SKB_CB(skb)->end_seq, start)) { 4796 tcp_collapse(sk, &tp->out_of_order_queue, 4797 head, skb, start, end); 4798 head = skb; 4799 if (!skb) 4800 break; 4801 /* Start new segment */ 4802 start = TCP_SKB_CB(skb)->seq; 4803 end = TCP_SKB_CB(skb)->end_seq; 4804 } else { 4805 if (before(TCP_SKB_CB(skb)->seq, start)) 4806 start = TCP_SKB_CB(skb)->seq; 4807 if (after(TCP_SKB_CB(skb)->end_seq, end)) 4808 end = TCP_SKB_CB(skb)->end_seq; 4809 } 4810 } 4811 } 4812 4813 /* 4814 * Purge the out-of-order queue. 4815 * Return true if queue was pruned. 4816 */ 4817 static bool tcp_prune_ofo_queue(struct sock *sk) 4818 { 4819 struct tcp_sock *tp = tcp_sk(sk); 4820 bool res = false; 4821 4822 if (!skb_queue_empty(&tp->out_of_order_queue)) { 4823 NET_INC_STATS_BH(sock_net(sk), LINUX_MIB_OFOPRUNED); 4824 __skb_queue_purge(&tp->out_of_order_queue); 4825 4826 /* Reset SACK state. A conforming SACK implementation will 4827 * do the same at a timeout based retransmit. When a connection 4828 * is in a sad state like this, we care only about integrity 4829 * of the connection not performance. 4830 */ 4831 if (tp->rx_opt.sack_ok) 4832 tcp_sack_reset(&tp->rx_opt); 4833 sk_mem_reclaim(sk); 4834 res = true; 4835 } 4836 return res; 4837 } 4838 4839 /* Reduce allocated memory if we can, trying to get 4840 * the socket within its memory limits again. 4841 * 4842 * Return less than zero if we should start dropping frames 4843 * until the socket owning process reads some of the data 4844 * to stabilize the situation. 4845 */ 4846 static int tcp_prune_queue(struct sock *sk) 4847 { 4848 struct tcp_sock *tp = tcp_sk(sk); 4849 4850 SOCK_DEBUG(sk, "prune_queue: c=%x\n", tp->copied_seq); 4851 4852 NET_INC_STATS_BH(sock_net(sk), LINUX_MIB_PRUNECALLED); 4853 4854 if (atomic_read(&sk->sk_rmem_alloc) >= sk->sk_rcvbuf) 4855 tcp_clamp_window(sk); 4856 else if (tcp_under_memory_pressure(sk)) 4857 tp->rcv_ssthresh = min(tp->rcv_ssthresh, 4U * tp->advmss); 4858 4859 tcp_collapse_ofo_queue(sk); 4860 if (!skb_queue_empty(&sk->sk_receive_queue)) 4861 tcp_collapse(sk, &sk->sk_receive_queue, 4862 skb_peek(&sk->sk_receive_queue), 4863 NULL, 4864 tp->copied_seq, tp->rcv_nxt); 4865 sk_mem_reclaim(sk); 4866 4867 if (atomic_read(&sk->sk_rmem_alloc) <= sk->sk_rcvbuf) 4868 return 0; 4869 4870 /* Collapsing did not help, destructive actions follow. 4871 * This must not ever occur. */ 4872 4873 tcp_prune_ofo_queue(sk); 4874 4875 if (atomic_read(&sk->sk_rmem_alloc) <= sk->sk_rcvbuf) 4876 return 0; 4877 4878 /* If we are really being abused, tell the caller to silently 4879 * drop receive data on the floor. It will get retransmitted 4880 * and hopefully then we'll have sufficient space. 4881 */ 4882 NET_INC_STATS_BH(sock_net(sk), LINUX_MIB_RCVPRUNED); 4883 4884 /* Massive buffer overcommit. */ 4885 tp->pred_flags = 0; 4886 return -1; 4887 } 4888 4889 static bool tcp_should_expand_sndbuf(const struct sock *sk) 4890 { 4891 const struct tcp_sock *tp = tcp_sk(sk); 4892 4893 /* If the user specified a specific send buffer setting, do 4894 * not modify it. 4895 */ 4896 if (sk->sk_userlocks & SOCK_SNDBUF_LOCK) 4897 return false; 4898 4899 /* If we are under global TCP memory pressure, do not expand. */ 4900 if (tcp_under_memory_pressure(sk)) 4901 return false; 4902 4903 /* If we are under soft global TCP memory pressure, do not expand. */ 4904 if (sk_memory_allocated(sk) >= sk_prot_mem_limits(sk, 0)) 4905 return false; 4906 4907 /* If we filled the congestion window, do not expand. */ 4908 if (tcp_packets_in_flight(tp) >= tp->snd_cwnd) 4909 return false; 4910 4911 return true; 4912 } 4913 4914 /* When incoming ACK allowed to free some skb from write_queue, 4915 * we remember this event in flag SOCK_QUEUE_SHRUNK and wake up socket 4916 * on the exit from tcp input handler. 4917 * 4918 * PROBLEM: sndbuf expansion does not work well with largesend. 4919 */ 4920 static void tcp_new_space(struct sock *sk) 4921 { 4922 struct tcp_sock *tp = tcp_sk(sk); 4923 4924 if (tcp_should_expand_sndbuf(sk)) { 4925 tcp_sndbuf_expand(sk); 4926 tp->snd_cwnd_stamp = tcp_time_stamp; 4927 } 4928 4929 sk->sk_write_space(sk); 4930 } 4931 4932 static void tcp_check_space(struct sock *sk) 4933 { 4934 if (sock_flag(sk, SOCK_QUEUE_SHRUNK)) { 4935 sock_reset_flag(sk, SOCK_QUEUE_SHRUNK); 4936 /* pairs with tcp_poll() */ 4937 smp_mb__after_atomic(); 4938 if (sk->sk_socket && 4939 test_bit(SOCK_NOSPACE, &sk->sk_socket->flags)) 4940 tcp_new_space(sk); 4941 } 4942 } 4943 4944 static inline void tcp_data_snd_check(struct sock *sk) 4945 { 4946 tcp_push_pending_frames(sk); 4947 tcp_check_space(sk); 4948 } 4949 4950 /* 4951 * Check if sending an ack is needed. 4952 */ 4953 static void __tcp_ack_snd_check(struct sock *sk, int ofo_possible) 4954 { 4955 struct tcp_sock *tp = tcp_sk(sk); 4956 4957 /* More than one full frame received... */ 4958 if (((tp->rcv_nxt - tp->rcv_wup) > inet_csk(sk)->icsk_ack.rcv_mss && 4959 /* ... and right edge of window advances far enough. 4960 * (tcp_recvmsg() will send ACK otherwise). Or... 4961 */ 4962 __tcp_select_window(sk) >= tp->rcv_wnd) || 4963 /* We ACK each frame or... */ 4964 tcp_in_quickack_mode(sk) || 4965 /* We have out of order data. */ 4966 (ofo_possible && skb_peek(&tp->out_of_order_queue))) { 4967 /* Then ack it now */ 4968 tcp_send_ack(sk); 4969 } else { 4970 /* Else, send delayed ack. */ 4971 tcp_send_delayed_ack(sk); 4972 } 4973 } 4974 4975 static inline void tcp_ack_snd_check(struct sock *sk) 4976 { 4977 if (!inet_csk_ack_scheduled(sk)) { 4978 /* We sent a data segment already. */ 4979 return; 4980 } 4981 __tcp_ack_snd_check(sk, 1); 4982 } 4983 4984 /* 4985 * This routine is only called when we have urgent data 4986 * signaled. Its the 'slow' part of tcp_urg. It could be 4987 * moved inline now as tcp_urg is only called from one 4988 * place. We handle URGent data wrong. We have to - as 4989 * BSD still doesn't use the correction from RFC961. 4990 * For 1003.1g we should support a new option TCP_STDURG to permit 4991 * either form (or just set the sysctl tcp_stdurg). 4992 */ 4993 4994 static void tcp_check_urg(struct sock *sk, const struct tcphdr *th) 4995 { 4996 struct tcp_sock *tp = tcp_sk(sk); 4997 u32 ptr = ntohs(th->urg_ptr); 4998 4999 if (ptr && !sysctl_tcp_stdurg) 5000 ptr--; 5001 ptr += ntohl(th->seq); 5002 5003 /* Ignore urgent data that we've already seen and read. */ 5004 if (after(tp->copied_seq, ptr)) 5005 return; 5006 5007 /* Do not replay urg ptr. 5008 * 5009 * NOTE: interesting situation not covered by specs. 5010 * Misbehaving sender may send urg ptr, pointing to segment, 5011 * which we already have in ofo queue. We are not able to fetch 5012 * such data and will stay in TCP_URG_NOTYET until will be eaten 5013 * by recvmsg(). Seems, we are not obliged to handle such wicked 5014 * situations. But it is worth to think about possibility of some 5015 * DoSes using some hypothetical application level deadlock. 5016 */ 5017 if (before(ptr, tp->rcv_nxt)) 5018 return; 5019 5020 /* Do we already have a newer (or duplicate) urgent pointer? */ 5021 if (tp->urg_data && !after(ptr, tp->urg_seq)) 5022 return; 5023 5024 /* Tell the world about our new urgent pointer. */ 5025 sk_send_sigurg(sk); 5026 5027 /* We may be adding urgent data when the last byte read was 5028 * urgent. To do this requires some care. We cannot just ignore 5029 * tp->copied_seq since we would read the last urgent byte again 5030 * as data, nor can we alter copied_seq until this data arrives 5031 * or we break the semantics of SIOCATMARK (and thus sockatmark()) 5032 * 5033 * NOTE. Double Dutch. Rendering to plain English: author of comment 5034 * above did something sort of send("A", MSG_OOB); send("B", MSG_OOB); 5035 * and expect that both A and B disappear from stream. This is _wrong_. 5036 * Though this happens in BSD with high probability, this is occasional. 5037 * Any application relying on this is buggy. Note also, that fix "works" 5038 * only in this artificial test. Insert some normal data between A and B and we will 5039 * decline of BSD again. Verdict: it is better to remove to trap 5040 * buggy users. 5041 */ 5042 if (tp->urg_seq == tp->copied_seq && tp->urg_data && 5043 !sock_flag(sk, SOCK_URGINLINE) && tp->copied_seq != tp->rcv_nxt) { 5044 struct sk_buff *skb = skb_peek(&sk->sk_receive_queue); 5045 tp->copied_seq++; 5046 if (skb && !before(tp->copied_seq, TCP_SKB_CB(skb)->end_seq)) { 5047 __skb_unlink(skb, &sk->sk_receive_queue); 5048 __kfree_skb(skb); 5049 } 5050 } 5051 5052 tp->urg_data = TCP_URG_NOTYET; 5053 tp->urg_seq = ptr; 5054 5055 /* Disable header prediction. */ 5056 tp->pred_flags = 0; 5057 } 5058 5059 /* This is the 'fast' part of urgent handling. */ 5060 static void tcp_urg(struct sock *sk, struct sk_buff *skb, const struct tcphdr *th) 5061 { 5062 struct tcp_sock *tp = tcp_sk(sk); 5063 5064 /* Check if we get a new urgent pointer - normally not. */ 5065 if (th->urg) 5066 tcp_check_urg(sk, th); 5067 5068 /* Do we wait for any urgent data? - normally not... */ 5069 if (tp->urg_data == TCP_URG_NOTYET) { 5070 u32 ptr = tp->urg_seq - ntohl(th->seq) + (th->doff * 4) - 5071 th->syn; 5072 5073 /* Is the urgent pointer pointing into this packet? */ 5074 if (ptr < skb->len) { 5075 u8 tmp; 5076 if (skb_copy_bits(skb, ptr, &tmp, 1)) 5077 BUG(); 5078 tp->urg_data = TCP_URG_VALID | tmp; 5079 if (!sock_flag(sk, SOCK_DEAD)) 5080 sk->sk_data_ready(sk); 5081 } 5082 } 5083 } 5084 5085 static int tcp_copy_to_iovec(struct sock *sk, struct sk_buff *skb, int hlen) 5086 { 5087 struct tcp_sock *tp = tcp_sk(sk); 5088 int chunk = skb->len - hlen; 5089 int err; 5090 5091 local_bh_enable(); 5092 if (skb_csum_unnecessary(skb)) 5093 err = skb_copy_datagram_msg(skb, hlen, tp->ucopy.msg, chunk); 5094 else 5095 err = skb_copy_and_csum_datagram_msg(skb, hlen, tp->ucopy.msg); 5096 5097 if (!err) { 5098 tp->ucopy.len -= chunk; 5099 tp->copied_seq += chunk; 5100 tcp_rcv_space_adjust(sk); 5101 } 5102 5103 local_bh_disable(); 5104 return err; 5105 } 5106 5107 static __sum16 __tcp_checksum_complete_user(struct sock *sk, 5108 struct sk_buff *skb) 5109 { 5110 __sum16 result; 5111 5112 if (sock_owned_by_user(sk)) { 5113 local_bh_enable(); 5114 result = __tcp_checksum_complete(skb); 5115 local_bh_disable(); 5116 } else { 5117 result = __tcp_checksum_complete(skb); 5118 } 5119 return result; 5120 } 5121 5122 static inline bool tcp_checksum_complete_user(struct sock *sk, 5123 struct sk_buff *skb) 5124 { 5125 return !skb_csum_unnecessary(skb) && 5126 __tcp_checksum_complete_user(sk, skb); 5127 } 5128 5129 /* Does PAWS and seqno based validation of an incoming segment, flags will 5130 * play significant role here. 5131 */ 5132 static bool tcp_validate_incoming(struct sock *sk, struct sk_buff *skb, 5133 const struct tcphdr *th, int syn_inerr) 5134 { 5135 struct tcp_sock *tp = tcp_sk(sk); 5136 5137 /* RFC1323: H1. Apply PAWS check first. */ 5138 if (tcp_fast_parse_options(skb, th, tp) && tp->rx_opt.saw_tstamp && 5139 tcp_paws_discard(sk, skb)) { 5140 if (!th->rst) { 5141 NET_INC_STATS_BH(sock_net(sk), LINUX_MIB_PAWSESTABREJECTED); 5142 if (!tcp_oow_rate_limited(sock_net(sk), skb, 5143 LINUX_MIB_TCPACKSKIPPEDPAWS, 5144 &tp->last_oow_ack_time)) 5145 tcp_send_dupack(sk, skb); 5146 goto discard; 5147 } 5148 /* Reset is accepted even if it did not pass PAWS. */ 5149 } 5150 5151 /* Step 1: check sequence number */ 5152 if (!tcp_sequence(tp, TCP_SKB_CB(skb)->seq, TCP_SKB_CB(skb)->end_seq)) { 5153 /* RFC793, page 37: "In all states except SYN-SENT, all reset 5154 * (RST) segments are validated by checking their SEQ-fields." 5155 * And page 69: "If an incoming segment is not acceptable, 5156 * an acknowledgment should be sent in reply (unless the RST 5157 * bit is set, if so drop the segment and return)". 5158 */ 5159 if (!th->rst) { 5160 if (th->syn) 5161 goto syn_challenge; 5162 if (!tcp_oow_rate_limited(sock_net(sk), skb, 5163 LINUX_MIB_TCPACKSKIPPEDSEQ, 5164 &tp->last_oow_ack_time)) 5165 tcp_send_dupack(sk, skb); 5166 } 5167 goto discard; 5168 } 5169 5170 /* Step 2: check RST bit */ 5171 if (th->rst) { 5172 /* RFC 5961 3.2 : 5173 * If sequence number exactly matches RCV.NXT, then 5174 * RESET the connection 5175 * else 5176 * Send a challenge ACK 5177 */ 5178 if (TCP_SKB_CB(skb)->seq == tp->rcv_nxt) 5179 tcp_reset(sk); 5180 else 5181 tcp_send_challenge_ack(sk, skb); 5182 goto discard; 5183 } 5184 5185 /* step 3: check security and precedence [ignored] */ 5186 5187 /* step 4: Check for a SYN 5188 * RFC 5961 4.2 : Send a challenge ack 5189 */ 5190 if (th->syn) { 5191 syn_challenge: 5192 if (syn_inerr) 5193 TCP_INC_STATS_BH(sock_net(sk), TCP_MIB_INERRS); 5194 NET_INC_STATS_BH(sock_net(sk), LINUX_MIB_TCPSYNCHALLENGE); 5195 tcp_send_challenge_ack(sk, skb); 5196 goto discard; 5197 } 5198 5199 return true; 5200 5201 discard: 5202 __kfree_skb(skb); 5203 return false; 5204 } 5205 5206 /* 5207 * TCP receive function for the ESTABLISHED state. 5208 * 5209 * It is split into a fast path and a slow path. The fast path is 5210 * disabled when: 5211 * - A zero window was announced from us - zero window probing 5212 * is only handled properly in the slow path. 5213 * - Out of order segments arrived. 5214 * - Urgent data is expected. 5215 * - There is no buffer space left 5216 * - Unexpected TCP flags/window values/header lengths are received 5217 * (detected by checking the TCP header against pred_flags) 5218 * - Data is sent in both directions. Fast path only supports pure senders 5219 * or pure receivers (this means either the sequence number or the ack 5220 * value must stay constant) 5221 * - Unexpected TCP option. 5222 * 5223 * When these conditions are not satisfied it drops into a standard 5224 * receive procedure patterned after RFC793 to handle all cases. 5225 * The first three cases are guaranteed by proper pred_flags setting, 5226 * the rest is checked inline. Fast processing is turned on in 5227 * tcp_data_queue when everything is OK. 5228 */ 5229 void tcp_rcv_established(struct sock *sk, struct sk_buff *skb, 5230 const struct tcphdr *th, unsigned int len) 5231 { 5232 struct tcp_sock *tp = tcp_sk(sk); 5233 5234 if (unlikely(!sk->sk_rx_dst)) 5235 inet_csk(sk)->icsk_af_ops->sk_rx_dst_set(sk, skb); 5236 /* 5237 * Header prediction. 5238 * The code loosely follows the one in the famous 5239 * "30 instruction TCP receive" Van Jacobson mail. 5240 * 5241 * Van's trick is to deposit buffers into socket queue 5242 * on a device interrupt, to call tcp_recv function 5243 * on the receive process context and checksum and copy 5244 * the buffer to user space. smart... 5245 * 5246 * Our current scheme is not silly either but we take the 5247 * extra cost of the net_bh soft interrupt processing... 5248 * We do checksum and copy also but from device to kernel. 5249 */ 5250 5251 tp->rx_opt.saw_tstamp = 0; 5252 5253 /* pred_flags is 0xS?10 << 16 + snd_wnd 5254 * if header_prediction is to be made 5255 * 'S' will always be tp->tcp_header_len >> 2 5256 * '?' will be 0 for the fast path, otherwise pred_flags is 0 to 5257 * turn it off (when there are holes in the receive 5258 * space for instance) 5259 * PSH flag is ignored. 5260 */ 5261 5262 if ((tcp_flag_word(th) & TCP_HP_BITS) == tp->pred_flags && 5263 TCP_SKB_CB(skb)->seq == tp->rcv_nxt && 5264 !after(TCP_SKB_CB(skb)->ack_seq, tp->snd_nxt)) { 5265 int tcp_header_len = tp->tcp_header_len; 5266 5267 /* Timestamp header prediction: tcp_header_len 5268 * is automatically equal to th->doff*4 due to pred_flags 5269 * match. 5270 */ 5271 5272 /* Check timestamp */ 5273 if (tcp_header_len == sizeof(struct tcphdr) + TCPOLEN_TSTAMP_ALIGNED) { 5274 /* No? Slow path! */ 5275 if (!tcp_parse_aligned_timestamp(tp, th)) 5276 goto slow_path; 5277 5278 /* If PAWS failed, check it more carefully in slow path */ 5279 if ((s32)(tp->rx_opt.rcv_tsval - tp->rx_opt.ts_recent) < 0) 5280 goto slow_path; 5281 5282 /* DO NOT update ts_recent here, if checksum fails 5283 * and timestamp was corrupted part, it will result 5284 * in a hung connection since we will drop all 5285 * future packets due to the PAWS test. 5286 */ 5287 } 5288 5289 if (len <= tcp_header_len) { 5290 /* Bulk data transfer: sender */ 5291 if (len == tcp_header_len) { 5292 /* Predicted packet is in window by definition. 5293 * seq == rcv_nxt and rcv_wup <= rcv_nxt. 5294 * Hence, check seq<=rcv_wup reduces to: 5295 */ 5296 if (tcp_header_len == 5297 (sizeof(struct tcphdr) + TCPOLEN_TSTAMP_ALIGNED) && 5298 tp->rcv_nxt == tp->rcv_wup) 5299 tcp_store_ts_recent(tp); 5300 5301 /* We know that such packets are checksummed 5302 * on entry. 5303 */ 5304 tcp_ack(sk, skb, 0); 5305 __kfree_skb(skb); 5306 tcp_data_snd_check(sk); 5307 return; 5308 } else { /* Header too small */ 5309 TCP_INC_STATS_BH(sock_net(sk), TCP_MIB_INERRS); 5310 goto discard; 5311 } 5312 } else { 5313 int eaten = 0; 5314 bool fragstolen = false; 5315 5316 if (tp->ucopy.task == current && 5317 tp->copied_seq == tp->rcv_nxt && 5318 len - tcp_header_len <= tp->ucopy.len && 5319 sock_owned_by_user(sk)) { 5320 __set_current_state(TASK_RUNNING); 5321 5322 if (!tcp_copy_to_iovec(sk, skb, tcp_header_len)) { 5323 /* Predicted packet is in window by definition. 5324 * seq == rcv_nxt and rcv_wup <= rcv_nxt. 5325 * Hence, check seq<=rcv_wup reduces to: 5326 */ 5327 if (tcp_header_len == 5328 (sizeof(struct tcphdr) + 5329 TCPOLEN_TSTAMP_ALIGNED) && 5330 tp->rcv_nxt == tp->rcv_wup) 5331 tcp_store_ts_recent(tp); 5332 5333 tcp_rcv_rtt_measure_ts(sk, skb); 5334 5335 __skb_pull(skb, tcp_header_len); 5336 tcp_rcv_nxt_update(tp, TCP_SKB_CB(skb)->end_seq); 5337 NET_INC_STATS_BH(sock_net(sk), LINUX_MIB_TCPHPHITSTOUSER); 5338 eaten = 1; 5339 } 5340 } 5341 if (!eaten) { 5342 if (tcp_checksum_complete_user(sk, skb)) 5343 goto csum_error; 5344 5345 if ((int)skb->truesize > sk->sk_forward_alloc) 5346 goto step5; 5347 5348 /* Predicted packet is in window by definition. 5349 * seq == rcv_nxt and rcv_wup <= rcv_nxt. 5350 * Hence, check seq<=rcv_wup reduces to: 5351 */ 5352 if (tcp_header_len == 5353 (sizeof(struct tcphdr) + TCPOLEN_TSTAMP_ALIGNED) && 5354 tp->rcv_nxt == tp->rcv_wup) 5355 tcp_store_ts_recent(tp); 5356 5357 tcp_rcv_rtt_measure_ts(sk, skb); 5358 5359 NET_INC_STATS_BH(sock_net(sk), LINUX_MIB_TCPHPHITS); 5360 5361 /* Bulk data transfer: receiver */ 5362 eaten = tcp_queue_rcv(sk, skb, tcp_header_len, 5363 &fragstolen); 5364 } 5365 5366 tcp_event_data_recv(sk, skb); 5367 5368 if (TCP_SKB_CB(skb)->ack_seq != tp->snd_una) { 5369 /* Well, only one small jumplet in fast path... */ 5370 tcp_ack(sk, skb, FLAG_DATA); 5371 tcp_data_snd_check(sk); 5372 if (!inet_csk_ack_scheduled(sk)) 5373 goto no_ack; 5374 } 5375 5376 __tcp_ack_snd_check(sk, 0); 5377 no_ack: 5378 if (eaten) 5379 kfree_skb_partial(skb, fragstolen); 5380 sk->sk_data_ready(sk); 5381 return; 5382 } 5383 } 5384 5385 slow_path: 5386 if (len < (th->doff << 2) || tcp_checksum_complete_user(sk, skb)) 5387 goto csum_error; 5388 5389 if (!th->ack && !th->rst && !th->syn) 5390 goto discard; 5391 5392 /* 5393 * Standard slow path. 5394 */ 5395 5396 if (!tcp_validate_incoming(sk, skb, th, 1)) 5397 return; 5398 5399 step5: 5400 if (tcp_ack(sk, skb, FLAG_SLOWPATH | FLAG_UPDATE_TS_RECENT) < 0) 5401 goto discard; 5402 5403 tcp_rcv_rtt_measure_ts(sk, skb); 5404 5405 /* Process urgent data. */ 5406 tcp_urg(sk, skb, th); 5407 5408 /* step 7: process the segment text */ 5409 tcp_data_queue(sk, skb); 5410 5411 tcp_data_snd_check(sk); 5412 tcp_ack_snd_check(sk); 5413 return; 5414 5415 csum_error: 5416 TCP_INC_STATS_BH(sock_net(sk), TCP_MIB_CSUMERRORS); 5417 TCP_INC_STATS_BH(sock_net(sk), TCP_MIB_INERRS); 5418 5419 discard: 5420 __kfree_skb(skb); 5421 } 5422 EXPORT_SYMBOL(tcp_rcv_established); 5423 5424 void tcp_finish_connect(struct sock *sk, struct sk_buff *skb) 5425 { 5426 struct tcp_sock *tp = tcp_sk(sk); 5427 struct inet_connection_sock *icsk = inet_csk(sk); 5428 5429 tcp_set_state(sk, TCP_ESTABLISHED); 5430 5431 if (skb) { 5432 icsk->icsk_af_ops->sk_rx_dst_set(sk, skb); 5433 security_inet_conn_established(sk, skb); 5434 } 5435 5436 /* Make sure socket is routed, for correct metrics. */ 5437 icsk->icsk_af_ops->rebuild_header(sk); 5438 5439 tcp_init_metrics(sk); 5440 5441 tcp_init_congestion_control(sk); 5442 5443 /* Prevent spurious tcp_cwnd_restart() on first data 5444 * packet. 5445 */ 5446 tp->lsndtime = tcp_time_stamp; 5447 5448 tcp_init_buffer_space(sk); 5449 5450 if (sock_flag(sk, SOCK_KEEPOPEN)) 5451 inet_csk_reset_keepalive_timer(sk, keepalive_time_when(tp)); 5452 5453 if (!tp->rx_opt.snd_wscale) 5454 __tcp_fast_path_on(tp, tp->snd_wnd); 5455 else 5456 tp->pred_flags = 0; 5457 5458 if (!sock_flag(sk, SOCK_DEAD)) { 5459 sk->sk_state_change(sk); 5460 sk_wake_async(sk, SOCK_WAKE_IO, POLL_OUT); 5461 } 5462 } 5463 5464 static bool tcp_rcv_fastopen_synack(struct sock *sk, struct sk_buff *synack, 5465 struct tcp_fastopen_cookie *cookie) 5466 { 5467 struct tcp_sock *tp = tcp_sk(sk); 5468 struct sk_buff *data = tp->syn_data ? tcp_write_queue_head(sk) : NULL; 5469 u16 mss = tp->rx_opt.mss_clamp, try_exp = 0; 5470 bool syn_drop = false; 5471 5472 if (mss == tp->rx_opt.user_mss) { 5473 struct tcp_options_received opt; 5474 5475 /* Get original SYNACK MSS value if user MSS sets mss_clamp */ 5476 tcp_clear_options(&opt); 5477 opt.user_mss = opt.mss_clamp = 0; 5478 tcp_parse_options(synack, &opt, 0, NULL); 5479 mss = opt.mss_clamp; 5480 } 5481 5482 if (!tp->syn_fastopen) { 5483 /* Ignore an unsolicited cookie */ 5484 cookie->len = -1; 5485 } else if (tp->total_retrans) { 5486 /* SYN timed out and the SYN-ACK neither has a cookie nor 5487 * acknowledges data. Presumably the remote received only 5488 * the retransmitted (regular) SYNs: either the original 5489 * SYN-data or the corresponding SYN-ACK was dropped. 5490 */ 5491 syn_drop = (cookie->len < 0 && data); 5492 } else if (cookie->len < 0 && !tp->syn_data) { 5493 /* We requested a cookie but didn't get it. If we did not use 5494 * the (old) exp opt format then try so next time (try_exp=1). 5495 * Otherwise we go back to use the RFC7413 opt (try_exp=2). 5496 */ 5497 try_exp = tp->syn_fastopen_exp ? 2 : 1; 5498 } 5499 5500 tcp_fastopen_cache_set(sk, mss, cookie, syn_drop, try_exp); 5501 5502 if (data) { /* Retransmit unacked data in SYN */ 5503 tcp_for_write_queue_from(data, sk) { 5504 if (data == tcp_send_head(sk) || 5505 __tcp_retransmit_skb(sk, data)) 5506 break; 5507 } 5508 tcp_rearm_rto(sk); 5509 NET_INC_STATS_BH(sock_net(sk), LINUX_MIB_TCPFASTOPENACTIVEFAIL); 5510 return true; 5511 } 5512 tp->syn_data_acked = tp->syn_data; 5513 if (tp->syn_data_acked) 5514 NET_INC_STATS_BH(sock_net(sk), LINUX_MIB_TCPFASTOPENACTIVE); 5515 return false; 5516 } 5517 5518 static int tcp_rcv_synsent_state_process(struct sock *sk, struct sk_buff *skb, 5519 const struct tcphdr *th) 5520 { 5521 struct inet_connection_sock *icsk = inet_csk(sk); 5522 struct tcp_sock *tp = tcp_sk(sk); 5523 struct tcp_fastopen_cookie foc = { .len = -1 }; 5524 int saved_clamp = tp->rx_opt.mss_clamp; 5525 5526 tcp_parse_options(skb, &tp->rx_opt, 0, &foc); 5527 if (tp->rx_opt.saw_tstamp && tp->rx_opt.rcv_tsecr) 5528 tp->rx_opt.rcv_tsecr -= tp->tsoffset; 5529 5530 if (th->ack) { 5531 /* rfc793: 5532 * "If the state is SYN-SENT then 5533 * first check the ACK bit 5534 * If the ACK bit is set 5535 * If SEG.ACK =< ISS, or SEG.ACK > SND.NXT, send 5536 * a reset (unless the RST bit is set, if so drop 5537 * the segment and return)" 5538 */ 5539 if (!after(TCP_SKB_CB(skb)->ack_seq, tp->snd_una) || 5540 after(TCP_SKB_CB(skb)->ack_seq, tp->snd_nxt)) 5541 goto reset_and_undo; 5542 5543 if (tp->rx_opt.saw_tstamp && tp->rx_opt.rcv_tsecr && 5544 !between(tp->rx_opt.rcv_tsecr, tp->retrans_stamp, 5545 tcp_time_stamp)) { 5546 NET_INC_STATS_BH(sock_net(sk), LINUX_MIB_PAWSACTIVEREJECTED); 5547 goto reset_and_undo; 5548 } 5549 5550 /* Now ACK is acceptable. 5551 * 5552 * "If the RST bit is set 5553 * If the ACK was acceptable then signal the user "error: 5554 * connection reset", drop the segment, enter CLOSED state, 5555 * delete TCB, and return." 5556 */ 5557 5558 if (th->rst) { 5559 tcp_reset(sk); 5560 goto discard; 5561 } 5562 5563 /* rfc793: 5564 * "fifth, if neither of the SYN or RST bits is set then 5565 * drop the segment and return." 5566 * 5567 * See note below! 5568 * --ANK(990513) 5569 */ 5570 if (!th->syn) 5571 goto discard_and_undo; 5572 5573 /* rfc793: 5574 * "If the SYN bit is on ... 5575 * are acceptable then ... 5576 * (our SYN has been ACKed), change the connection 5577 * state to ESTABLISHED..." 5578 */ 5579 5580 tcp_ecn_rcv_synack(tp, th); 5581 5582 tcp_init_wl(tp, TCP_SKB_CB(skb)->seq); 5583 tcp_ack(sk, skb, FLAG_SLOWPATH); 5584 5585 /* Ok.. it's good. Set up sequence numbers and 5586 * move to established. 5587 */ 5588 tp->rcv_nxt = TCP_SKB_CB(skb)->seq + 1; 5589 tp->rcv_wup = TCP_SKB_CB(skb)->seq + 1; 5590 5591 /* RFC1323: The window in SYN & SYN/ACK segments is 5592 * never scaled. 5593 */ 5594 tp->snd_wnd = ntohs(th->window); 5595 5596 if (!tp->rx_opt.wscale_ok) { 5597 tp->rx_opt.snd_wscale = tp->rx_opt.rcv_wscale = 0; 5598 tp->window_clamp = min(tp->window_clamp, 65535U); 5599 } 5600 5601 if (tp->rx_opt.saw_tstamp) { 5602 tp->rx_opt.tstamp_ok = 1; 5603 tp->tcp_header_len = 5604 sizeof(struct tcphdr) + TCPOLEN_TSTAMP_ALIGNED; 5605 tp->advmss -= TCPOLEN_TSTAMP_ALIGNED; 5606 tcp_store_ts_recent(tp); 5607 } else { 5608 tp->tcp_header_len = sizeof(struct tcphdr); 5609 } 5610 5611 if (tcp_is_sack(tp) && sysctl_tcp_fack) 5612 tcp_enable_fack(tp); 5613 5614 tcp_mtup_init(sk); 5615 tcp_sync_mss(sk, icsk->icsk_pmtu_cookie); 5616 tcp_initialize_rcv_mss(sk); 5617 5618 /* Remember, tcp_poll() does not lock socket! 5619 * Change state from SYN-SENT only after copied_seq 5620 * is initialized. */ 5621 tp->copied_seq = tp->rcv_nxt; 5622 5623 smp_mb(); 5624 5625 tcp_finish_connect(sk, skb); 5626 5627 if ((tp->syn_fastopen || tp->syn_data) && 5628 tcp_rcv_fastopen_synack(sk, skb, &foc)) 5629 return -1; 5630 5631 if (sk->sk_write_pending || 5632 icsk->icsk_accept_queue.rskq_defer_accept || 5633 icsk->icsk_ack.pingpong) { 5634 /* Save one ACK. Data will be ready after 5635 * several ticks, if write_pending is set. 5636 * 5637 * It may be deleted, but with this feature tcpdumps 5638 * look so _wonderfully_ clever, that I was not able 5639 * to stand against the temptation 8) --ANK 5640 */ 5641 inet_csk_schedule_ack(sk); 5642 icsk->icsk_ack.lrcvtime = tcp_time_stamp; 5643 tcp_enter_quickack_mode(sk); 5644 inet_csk_reset_xmit_timer(sk, ICSK_TIME_DACK, 5645 TCP_DELACK_MAX, TCP_RTO_MAX); 5646 5647 discard: 5648 __kfree_skb(skb); 5649 return 0; 5650 } else { 5651 tcp_send_ack(sk); 5652 } 5653 return -1; 5654 } 5655 5656 /* No ACK in the segment */ 5657 5658 if (th->rst) { 5659 /* rfc793: 5660 * "If the RST bit is set 5661 * 5662 * Otherwise (no ACK) drop the segment and return." 5663 */ 5664 5665 goto discard_and_undo; 5666 } 5667 5668 /* PAWS check. */ 5669 if (tp->rx_opt.ts_recent_stamp && tp->rx_opt.saw_tstamp && 5670 tcp_paws_reject(&tp->rx_opt, 0)) 5671 goto discard_and_undo; 5672 5673 if (th->syn) { 5674 /* We see SYN without ACK. It is attempt of 5675 * simultaneous connect with crossed SYNs. 5676 * Particularly, it can be connect to self. 5677 */ 5678 tcp_set_state(sk, TCP_SYN_RECV); 5679 5680 if (tp->rx_opt.saw_tstamp) { 5681 tp->rx_opt.tstamp_ok = 1; 5682 tcp_store_ts_recent(tp); 5683 tp->tcp_header_len = 5684 sizeof(struct tcphdr) + TCPOLEN_TSTAMP_ALIGNED; 5685 } else { 5686 tp->tcp_header_len = sizeof(struct tcphdr); 5687 } 5688 5689 tp->rcv_nxt = TCP_SKB_CB(skb)->seq + 1; 5690 tp->copied_seq = tp->rcv_nxt; 5691 tp->rcv_wup = TCP_SKB_CB(skb)->seq + 1; 5692 5693 /* RFC1323: The window in SYN & SYN/ACK segments is 5694 * never scaled. 5695 */ 5696 tp->snd_wnd = ntohs(th->window); 5697 tp->snd_wl1 = TCP_SKB_CB(skb)->seq; 5698 tp->max_window = tp->snd_wnd; 5699 5700 tcp_ecn_rcv_syn(tp, th); 5701 5702 tcp_mtup_init(sk); 5703 tcp_sync_mss(sk, icsk->icsk_pmtu_cookie); 5704 tcp_initialize_rcv_mss(sk); 5705 5706 tcp_send_synack(sk); 5707 #if 0 5708 /* Note, we could accept data and URG from this segment. 5709 * There are no obstacles to make this (except that we must 5710 * either change tcp_recvmsg() to prevent it from returning data 5711 * before 3WHS completes per RFC793, or employ TCP Fast Open). 5712 * 5713 * However, if we ignore data in ACKless segments sometimes, 5714 * we have no reasons to accept it sometimes. 5715 * Also, seems the code doing it in step6 of tcp_rcv_state_process 5716 * is not flawless. So, discard packet for sanity. 5717 * Uncomment this return to process the data. 5718 */ 5719 return -1; 5720 #else 5721 goto discard; 5722 #endif 5723 } 5724 /* "fifth, if neither of the SYN or RST bits is set then 5725 * drop the segment and return." 5726 */ 5727 5728 discard_and_undo: 5729 tcp_clear_options(&tp->rx_opt); 5730 tp->rx_opt.mss_clamp = saved_clamp; 5731 goto discard; 5732 5733 reset_and_undo: 5734 tcp_clear_options(&tp->rx_opt); 5735 tp->rx_opt.mss_clamp = saved_clamp; 5736 return 1; 5737 } 5738 5739 /* 5740 * This function implements the receiving procedure of RFC 793 for 5741 * all states except ESTABLISHED and TIME_WAIT. 5742 * It's called from both tcp_v4_rcv and tcp_v6_rcv and should be 5743 * address independent. 5744 */ 5745 5746 int tcp_rcv_state_process(struct sock *sk, struct sk_buff *skb) 5747 { 5748 struct tcp_sock *tp = tcp_sk(sk); 5749 struct inet_connection_sock *icsk = inet_csk(sk); 5750 const struct tcphdr *th = tcp_hdr(skb); 5751 struct request_sock *req; 5752 int queued = 0; 5753 bool acceptable; 5754 5755 tp->rx_opt.saw_tstamp = 0; 5756 5757 switch (sk->sk_state) { 5758 case TCP_CLOSE: 5759 goto discard; 5760 5761 case TCP_LISTEN: 5762 if (th->ack) 5763 return 1; 5764 5765 if (th->rst) 5766 goto discard; 5767 5768 if (th->syn) { 5769 if (th->fin) 5770 goto discard; 5771 if (icsk->icsk_af_ops->conn_request(sk, skb) < 0) 5772 return 1; 5773 5774 /* Now we have several options: In theory there is 5775 * nothing else in the frame. KA9Q has an option to 5776 * send data with the syn, BSD accepts data with the 5777 * syn up to the [to be] advertised window and 5778 * Solaris 2.1 gives you a protocol error. For now 5779 * we just ignore it, that fits the spec precisely 5780 * and avoids incompatibilities. It would be nice in 5781 * future to drop through and process the data. 5782 * 5783 * Now that TTCP is starting to be used we ought to 5784 * queue this data. 5785 * But, this leaves one open to an easy denial of 5786 * service attack, and SYN cookies can't defend 5787 * against this problem. So, we drop the data 5788 * in the interest of security over speed unless 5789 * it's still in use. 5790 */ 5791 kfree_skb(skb); 5792 return 0; 5793 } 5794 goto discard; 5795 5796 case TCP_SYN_SENT: 5797 queued = tcp_rcv_synsent_state_process(sk, skb, th); 5798 if (queued >= 0) 5799 return queued; 5800 5801 /* Do step6 onward by hand. */ 5802 tcp_urg(sk, skb, th); 5803 __kfree_skb(skb); 5804 tcp_data_snd_check(sk); 5805 return 0; 5806 } 5807 5808 req = tp->fastopen_rsk; 5809 if (req) { 5810 WARN_ON_ONCE(sk->sk_state != TCP_SYN_RECV && 5811 sk->sk_state != TCP_FIN_WAIT1); 5812 5813 if (!tcp_check_req(sk, skb, req, true)) 5814 goto discard; 5815 } 5816 5817 if (!th->ack && !th->rst && !th->syn) 5818 goto discard; 5819 5820 if (!tcp_validate_incoming(sk, skb, th, 0)) 5821 return 0; 5822 5823 /* step 5: check the ACK field */ 5824 acceptable = tcp_ack(sk, skb, FLAG_SLOWPATH | 5825 FLAG_UPDATE_TS_RECENT) > 0; 5826 5827 switch (sk->sk_state) { 5828 case TCP_SYN_RECV: 5829 if (!acceptable) 5830 return 1; 5831 5832 if (!tp->srtt_us) 5833 tcp_synack_rtt_meas(sk, req); 5834 5835 /* Once we leave TCP_SYN_RECV, we no longer need req 5836 * so release it. 5837 */ 5838 if (req) { 5839 tp->total_retrans = req->num_retrans; 5840 reqsk_fastopen_remove(sk, req, false); 5841 } else { 5842 /* Make sure socket is routed, for correct metrics. */ 5843 icsk->icsk_af_ops->rebuild_header(sk); 5844 tcp_init_congestion_control(sk); 5845 5846 tcp_mtup_init(sk); 5847 tp->copied_seq = tp->rcv_nxt; 5848 tcp_init_buffer_space(sk); 5849 } 5850 smp_mb(); 5851 tcp_set_state(sk, TCP_ESTABLISHED); 5852 sk->sk_state_change(sk); 5853 5854 /* Note, that this wakeup is only for marginal crossed SYN case. 5855 * Passively open sockets are not waked up, because 5856 * sk->sk_sleep == NULL and sk->sk_socket == NULL. 5857 */ 5858 if (sk->sk_socket) 5859 sk_wake_async(sk, SOCK_WAKE_IO, POLL_OUT); 5860 5861 tp->snd_una = TCP_SKB_CB(skb)->ack_seq; 5862 tp->snd_wnd = ntohs(th->window) << tp->rx_opt.snd_wscale; 5863 tcp_init_wl(tp, TCP_SKB_CB(skb)->seq); 5864 5865 if (tp->rx_opt.tstamp_ok) 5866 tp->advmss -= TCPOLEN_TSTAMP_ALIGNED; 5867 5868 if (req) { 5869 /* Re-arm the timer because data may have been sent out. 5870 * This is similar to the regular data transmission case 5871 * when new data has just been ack'ed. 5872 * 5873 * (TFO) - we could try to be more aggressive and 5874 * retransmitting any data sooner based on when they 5875 * are sent out. 5876 */ 5877 tcp_rearm_rto(sk); 5878 } else 5879 tcp_init_metrics(sk); 5880 5881 tcp_update_pacing_rate(sk); 5882 5883 /* Prevent spurious tcp_cwnd_restart() on first data packet */ 5884 tp->lsndtime = tcp_time_stamp; 5885 5886 tcp_initialize_rcv_mss(sk); 5887 tcp_fast_path_on(tp); 5888 break; 5889 5890 case TCP_FIN_WAIT1: { 5891 struct dst_entry *dst; 5892 int tmo; 5893 5894 /* If we enter the TCP_FIN_WAIT1 state and we are a 5895 * Fast Open socket and this is the first acceptable 5896 * ACK we have received, this would have acknowledged 5897 * our SYNACK so stop the SYNACK timer. 5898 */ 5899 if (req) { 5900 /* Return RST if ack_seq is invalid. 5901 * Note that RFC793 only says to generate a 5902 * DUPACK for it but for TCP Fast Open it seems 5903 * better to treat this case like TCP_SYN_RECV 5904 * above. 5905 */ 5906 if (!acceptable) 5907 return 1; 5908 /* We no longer need the request sock. */ 5909 reqsk_fastopen_remove(sk, req, false); 5910 tcp_rearm_rto(sk); 5911 } 5912 if (tp->snd_una != tp->write_seq) 5913 break; 5914 5915 tcp_set_state(sk, TCP_FIN_WAIT2); 5916 sk->sk_shutdown |= SEND_SHUTDOWN; 5917 5918 dst = __sk_dst_get(sk); 5919 if (dst) 5920 dst_confirm(dst); 5921 5922 if (!sock_flag(sk, SOCK_DEAD)) { 5923 /* Wake up lingering close() */ 5924 sk->sk_state_change(sk); 5925 break; 5926 } 5927 5928 if (tp->linger2 < 0 || 5929 (TCP_SKB_CB(skb)->end_seq != TCP_SKB_CB(skb)->seq && 5930 after(TCP_SKB_CB(skb)->end_seq - th->fin, tp->rcv_nxt))) { 5931 tcp_done(sk); 5932 NET_INC_STATS_BH(sock_net(sk), LINUX_MIB_TCPABORTONDATA); 5933 return 1; 5934 } 5935 5936 tmo = tcp_fin_time(sk); 5937 if (tmo > TCP_TIMEWAIT_LEN) { 5938 inet_csk_reset_keepalive_timer(sk, tmo - TCP_TIMEWAIT_LEN); 5939 } else if (th->fin || sock_owned_by_user(sk)) { 5940 /* Bad case. We could lose such FIN otherwise. 5941 * It is not a big problem, but it looks confusing 5942 * and not so rare event. We still can lose it now, 5943 * if it spins in bh_lock_sock(), but it is really 5944 * marginal case. 5945 */ 5946 inet_csk_reset_keepalive_timer(sk, tmo); 5947 } else { 5948 tcp_time_wait(sk, TCP_FIN_WAIT2, tmo); 5949 goto discard; 5950 } 5951 break; 5952 } 5953 5954 case TCP_CLOSING: 5955 if (tp->snd_una == tp->write_seq) { 5956 tcp_time_wait(sk, TCP_TIME_WAIT, 0); 5957 goto discard; 5958 } 5959 break; 5960 5961 case TCP_LAST_ACK: 5962 if (tp->snd_una == tp->write_seq) { 5963 tcp_update_metrics(sk); 5964 tcp_done(sk); 5965 goto discard; 5966 } 5967 break; 5968 } 5969 5970 /* step 6: check the URG bit */ 5971 tcp_urg(sk, skb, th); 5972 5973 /* step 7: process the segment text */ 5974 switch (sk->sk_state) { 5975 case TCP_CLOSE_WAIT: 5976 case TCP_CLOSING: 5977 case TCP_LAST_ACK: 5978 if (!before(TCP_SKB_CB(skb)->seq, tp->rcv_nxt)) 5979 break; 5980 case TCP_FIN_WAIT1: 5981 case TCP_FIN_WAIT2: 5982 /* RFC 793 says to queue data in these states, 5983 * RFC 1122 says we MUST send a reset. 5984 * BSD 4.4 also does reset. 5985 */ 5986 if (sk->sk_shutdown & RCV_SHUTDOWN) { 5987 if (TCP_SKB_CB(skb)->end_seq != TCP_SKB_CB(skb)->seq && 5988 after(TCP_SKB_CB(skb)->end_seq - th->fin, tp->rcv_nxt)) { 5989 NET_INC_STATS_BH(sock_net(sk), LINUX_MIB_TCPABORTONDATA); 5990 tcp_reset(sk); 5991 return 1; 5992 } 5993 } 5994 /* Fall through */ 5995 case TCP_ESTABLISHED: 5996 tcp_data_queue(sk, skb); 5997 queued = 1; 5998 break; 5999 } 6000 6001 /* tcp_data could move socket to TIME-WAIT */ 6002 if (sk->sk_state != TCP_CLOSE) { 6003 tcp_data_snd_check(sk); 6004 tcp_ack_snd_check(sk); 6005 } 6006 6007 if (!queued) { 6008 discard: 6009 __kfree_skb(skb); 6010 } 6011 return 0; 6012 } 6013 EXPORT_SYMBOL(tcp_rcv_state_process); 6014 6015 static inline void pr_drop_req(struct request_sock *req, __u16 port, int family) 6016 { 6017 struct inet_request_sock *ireq = inet_rsk(req); 6018 6019 if (family == AF_INET) 6020 net_dbg_ratelimited("drop open request from %pI4/%u\n", 6021 &ireq->ir_rmt_addr, port); 6022 #if IS_ENABLED(CONFIG_IPV6) 6023 else if (family == AF_INET6) 6024 net_dbg_ratelimited("drop open request from %pI6/%u\n", 6025 &ireq->ir_v6_rmt_addr, port); 6026 #endif 6027 } 6028 6029 /* RFC3168 : 6.1.1 SYN packets must not have ECT/ECN bits set 6030 * 6031 * If we receive a SYN packet with these bits set, it means a 6032 * network is playing bad games with TOS bits. In order to 6033 * avoid possible false congestion notifications, we disable 6034 * TCP ECN negotiation. 6035 * 6036 * Exception: tcp_ca wants ECN. This is required for DCTCP 6037 * congestion control: Linux DCTCP asserts ECT on all packets, 6038 * including SYN, which is most optimal solution; however, 6039 * others, such as FreeBSD do not. 6040 */ 6041 static void tcp_ecn_create_request(struct request_sock *req, 6042 const struct sk_buff *skb, 6043 const struct sock *listen_sk, 6044 const struct dst_entry *dst) 6045 { 6046 const struct tcphdr *th = tcp_hdr(skb); 6047 const struct net *net = sock_net(listen_sk); 6048 bool th_ecn = th->ece && th->cwr; 6049 bool ect, ecn_ok; 6050 u32 ecn_ok_dst; 6051 6052 if (!th_ecn) 6053 return; 6054 6055 ect = !INET_ECN_is_not_ect(TCP_SKB_CB(skb)->ip_dsfield); 6056 ecn_ok_dst = dst_feature(dst, DST_FEATURE_ECN_MASK); 6057 ecn_ok = net->ipv4.sysctl_tcp_ecn || ecn_ok_dst; 6058 6059 if ((!ect && ecn_ok) || tcp_ca_needs_ecn(listen_sk) || 6060 (ecn_ok_dst & DST_FEATURE_ECN_CA)) 6061 inet_rsk(req)->ecn_ok = 1; 6062 } 6063 6064 static void tcp_openreq_init(struct request_sock *req, 6065 const struct tcp_options_received *rx_opt, 6066 struct sk_buff *skb, const struct sock *sk) 6067 { 6068 struct inet_request_sock *ireq = inet_rsk(req); 6069 6070 req->rsk_rcv_wnd = 0; /* So that tcp_send_synack() knows! */ 6071 req->cookie_ts = 0; 6072 tcp_rsk(req)->rcv_isn = TCP_SKB_CB(skb)->seq; 6073 tcp_rsk(req)->rcv_nxt = TCP_SKB_CB(skb)->seq + 1; 6074 skb_mstamp_get(&tcp_rsk(req)->snt_synack); 6075 tcp_rsk(req)->last_oow_ack_time = 0; 6076 req->mss = rx_opt->mss_clamp; 6077 req->ts_recent = rx_opt->saw_tstamp ? rx_opt->rcv_tsval : 0; 6078 ireq->tstamp_ok = rx_opt->tstamp_ok; 6079 ireq->sack_ok = rx_opt->sack_ok; 6080 ireq->snd_wscale = rx_opt->snd_wscale; 6081 ireq->wscale_ok = rx_opt->wscale_ok; 6082 ireq->acked = 0; 6083 ireq->ecn_ok = 0; 6084 ireq->ir_rmt_port = tcp_hdr(skb)->source; 6085 ireq->ir_num = ntohs(tcp_hdr(skb)->dest); 6086 ireq->ir_mark = inet_request_mark(sk, skb); 6087 } 6088 6089 struct request_sock *inet_reqsk_alloc(const struct request_sock_ops *ops, 6090 struct sock *sk_listener, 6091 bool attach_listener) 6092 { 6093 struct request_sock *req = reqsk_alloc(ops, sk_listener, 6094 attach_listener); 6095 6096 if (req) { 6097 struct inet_request_sock *ireq = inet_rsk(req); 6098 6099 kmemcheck_annotate_bitfield(ireq, flags); 6100 ireq->opt = NULL; 6101 atomic64_set(&ireq->ir_cookie, 0); 6102 ireq->ireq_state = TCP_NEW_SYN_RECV; 6103 write_pnet(&ireq->ireq_net, sock_net(sk_listener)); 6104 ireq->ireq_family = sk_listener->sk_family; 6105 } 6106 6107 return req; 6108 } 6109 EXPORT_SYMBOL(inet_reqsk_alloc); 6110 6111 /* 6112 * Return true if a syncookie should be sent 6113 */ 6114 static bool tcp_syn_flood_action(const struct sock *sk, 6115 const struct sk_buff *skb, 6116 const char *proto) 6117 { 6118 struct request_sock_queue *queue = &inet_csk(sk)->icsk_accept_queue; 6119 const char *msg = "Dropping request"; 6120 bool want_cookie = false; 6121 6122 #ifdef CONFIG_SYN_COOKIES 6123 if (sysctl_tcp_syncookies) { 6124 msg = "Sending cookies"; 6125 want_cookie = true; 6126 NET_INC_STATS_BH(sock_net(sk), LINUX_MIB_TCPREQQFULLDOCOOKIES); 6127 } else 6128 #endif 6129 NET_INC_STATS_BH(sock_net(sk), LINUX_MIB_TCPREQQFULLDROP); 6130 6131 if (!queue->synflood_warned && 6132 sysctl_tcp_syncookies != 2 && 6133 xchg(&queue->synflood_warned, 1) == 0) 6134 pr_info("%s: Possible SYN flooding on port %d. %s. Check SNMP counters.\n", 6135 proto, ntohs(tcp_hdr(skb)->dest), msg); 6136 6137 return want_cookie; 6138 } 6139 6140 static void tcp_reqsk_record_syn(const struct sock *sk, 6141 struct request_sock *req, 6142 const struct sk_buff *skb) 6143 { 6144 if (tcp_sk(sk)->save_syn) { 6145 u32 len = skb_network_header_len(skb) + tcp_hdrlen(skb); 6146 u32 *copy; 6147 6148 copy = kmalloc(len + sizeof(u32), GFP_ATOMIC); 6149 if (copy) { 6150 copy[0] = len; 6151 memcpy(©[1], skb_network_header(skb), len); 6152 req->saved_syn = copy; 6153 } 6154 } 6155 } 6156 6157 int tcp_conn_request(struct request_sock_ops *rsk_ops, 6158 const struct tcp_request_sock_ops *af_ops, 6159 struct sock *sk, struct sk_buff *skb) 6160 { 6161 struct tcp_fastopen_cookie foc = { .len = -1 }; 6162 __u32 isn = TCP_SKB_CB(skb)->tcp_tw_isn; 6163 struct tcp_options_received tmp_opt; 6164 struct tcp_sock *tp = tcp_sk(sk); 6165 struct sock *fastopen_sk = NULL; 6166 struct dst_entry *dst = NULL; 6167 struct request_sock *req; 6168 bool want_cookie = false; 6169 struct flowi fl; 6170 6171 /* TW buckets are converted to open requests without 6172 * limitations, they conserve resources and peer is 6173 * evidently real one. 6174 */ 6175 if ((sysctl_tcp_syncookies == 2 || 6176 inet_csk_reqsk_queue_is_full(sk)) && !isn) { 6177 want_cookie = tcp_syn_flood_action(sk, skb, rsk_ops->slab_name); 6178 if (!want_cookie) 6179 goto drop; 6180 } 6181 6182 6183 /* Accept backlog is full. If we have already queued enough 6184 * of warm entries in syn queue, drop request. It is better than 6185 * clogging syn queue with openreqs with exponentially increasing 6186 * timeout. 6187 */ 6188 if (sk_acceptq_is_full(sk) && inet_csk_reqsk_queue_young(sk) > 1) { 6189 NET_INC_STATS_BH(sock_net(sk), LINUX_MIB_LISTENOVERFLOWS); 6190 goto drop; 6191 } 6192 6193 req = inet_reqsk_alloc(rsk_ops, sk, !want_cookie); 6194 if (!req) 6195 goto drop; 6196 6197 tcp_rsk(req)->af_specific = af_ops; 6198 6199 tcp_clear_options(&tmp_opt); 6200 tmp_opt.mss_clamp = af_ops->mss_clamp; 6201 tmp_opt.user_mss = tp->rx_opt.user_mss; 6202 tcp_parse_options(skb, &tmp_opt, 0, want_cookie ? NULL : &foc); 6203 6204 if (want_cookie && !tmp_opt.saw_tstamp) 6205 tcp_clear_options(&tmp_opt); 6206 6207 tmp_opt.tstamp_ok = tmp_opt.saw_tstamp; 6208 tcp_openreq_init(req, &tmp_opt, skb, sk); 6209 6210 /* Note: tcp_v6_init_req() might override ir_iif for link locals */ 6211 inet_rsk(req)->ir_iif = inet_request_bound_dev_if(sk, skb); 6212 6213 af_ops->init_req(req, sk, skb); 6214 6215 if (security_inet_conn_request(sk, skb, req)) 6216 goto drop_and_free; 6217 6218 if (!want_cookie && !isn) { 6219 /* VJ's idea. We save last timestamp seen 6220 * from the destination in peer table, when entering 6221 * state TIME-WAIT, and check against it before 6222 * accepting new connection request. 6223 * 6224 * If "isn" is not zero, this request hit alive 6225 * timewait bucket, so that all the necessary checks 6226 * are made in the function processing timewait state. 6227 */ 6228 if (tcp_death_row.sysctl_tw_recycle) { 6229 bool strict; 6230 6231 dst = af_ops->route_req(sk, &fl, req, &strict); 6232 6233 if (dst && strict && 6234 !tcp_peer_is_proven(req, dst, true, 6235 tmp_opt.saw_tstamp)) { 6236 NET_INC_STATS_BH(sock_net(sk), LINUX_MIB_PAWSPASSIVEREJECTED); 6237 goto drop_and_release; 6238 } 6239 } 6240 /* Kill the following clause, if you dislike this way. */ 6241 else if (!sysctl_tcp_syncookies && 6242 (sysctl_max_syn_backlog - inet_csk_reqsk_queue_len(sk) < 6243 (sysctl_max_syn_backlog >> 2)) && 6244 !tcp_peer_is_proven(req, dst, false, 6245 tmp_opt.saw_tstamp)) { 6246 /* Without syncookies last quarter of 6247 * backlog is filled with destinations, 6248 * proven to be alive. 6249 * It means that we continue to communicate 6250 * to destinations, already remembered 6251 * to the moment of synflood. 6252 */ 6253 pr_drop_req(req, ntohs(tcp_hdr(skb)->source), 6254 rsk_ops->family); 6255 goto drop_and_release; 6256 } 6257 6258 isn = af_ops->init_seq(skb); 6259 } 6260 if (!dst) { 6261 dst = af_ops->route_req(sk, &fl, req, NULL); 6262 if (!dst) 6263 goto drop_and_free; 6264 } 6265 6266 tcp_ecn_create_request(req, skb, sk, dst); 6267 6268 if (want_cookie) { 6269 isn = cookie_init_sequence(af_ops, sk, skb, &req->mss); 6270 req->cookie_ts = tmp_opt.tstamp_ok; 6271 if (!tmp_opt.tstamp_ok) 6272 inet_rsk(req)->ecn_ok = 0; 6273 } 6274 6275 tcp_rsk(req)->snt_isn = isn; 6276 tcp_rsk(req)->txhash = net_tx_rndhash(); 6277 tcp_openreq_init_rwin(req, sk, dst); 6278 if (!want_cookie) { 6279 tcp_reqsk_record_syn(sk, req, skb); 6280 fastopen_sk = tcp_try_fastopen(sk, skb, req, &foc, dst); 6281 } 6282 if (fastopen_sk) { 6283 af_ops->send_synack(fastopen_sk, dst, &fl, req, 6284 &foc, false); 6285 /* Add the child socket directly into the accept queue */ 6286 inet_csk_reqsk_queue_add(sk, req, fastopen_sk); 6287 sk->sk_data_ready(sk); 6288 bh_unlock_sock(fastopen_sk); 6289 sock_put(fastopen_sk); 6290 } else { 6291 tcp_rsk(req)->tfo_listener = false; 6292 if (!want_cookie) 6293 inet_csk_reqsk_queue_hash_add(sk, req, TCP_TIMEOUT_INIT); 6294 af_ops->send_synack(sk, dst, &fl, req, 6295 &foc, !want_cookie); 6296 if (want_cookie) 6297 goto drop_and_free; 6298 } 6299 reqsk_put(req); 6300 return 0; 6301 6302 drop_and_release: 6303 dst_release(dst); 6304 drop_and_free: 6305 reqsk_free(req); 6306 drop: 6307 NET_INC_STATS_BH(sock_net(sk), LINUX_MIB_LISTENDROPS); 6308 return 0; 6309 } 6310 EXPORT_SYMBOL(tcp_conn_request); 6311