1 /* 2 * INET An implementation of the TCP/IP protocol suite for the LINUX 3 * operating system. INET is implemented using the BSD Socket 4 * interface as the means of communication with the user level. 5 * 6 * Implementation of the Transmission Control Protocol(TCP). 7 * 8 * Authors: Ross Biro 9 * Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG> 10 * Mark Evans, <evansmp@uhura.aston.ac.uk> 11 * Corey Minyard <wf-rch!minyard@relay.EU.net> 12 * Florian La Roche, <flla@stud.uni-sb.de> 13 * Charles Hedrick, <hedrick@klinzhai.rutgers.edu> 14 * Linus Torvalds, <torvalds@cs.helsinki.fi> 15 * Alan Cox, <gw4pts@gw4pts.ampr.org> 16 * Matthew Dillon, <dillon@apollo.west.oic.com> 17 * Arnt Gulbrandsen, <agulbra@nvg.unit.no> 18 * Jorge Cwik, <jorge@laser.satlink.net> 19 */ 20 21 /* 22 * Changes: 23 * Pedro Roque : Fast Retransmit/Recovery. 24 * Two receive queues. 25 * Retransmit queue handled by TCP. 26 * Better retransmit timer handling. 27 * New congestion avoidance. 28 * Header prediction. 29 * Variable renaming. 30 * 31 * Eric : Fast Retransmit. 32 * Randy Scott : MSS option defines. 33 * Eric Schenk : Fixes to slow start algorithm. 34 * Eric Schenk : Yet another double ACK bug. 35 * Eric Schenk : Delayed ACK bug fixes. 36 * Eric Schenk : Floyd style fast retrans war avoidance. 37 * David S. Miller : Don't allow zero congestion window. 38 * Eric Schenk : Fix retransmitter so that it sends 39 * next packet on ack of previous packet. 40 * Andi Kleen : Moved open_request checking here 41 * and process RSTs for open_requests. 42 * Andi Kleen : Better prune_queue, and other fixes. 43 * Andrey Savochkin: Fix RTT measurements in the presence of 44 * timestamps. 45 * Andrey Savochkin: Check sequence numbers correctly when 46 * removing SACKs due to in sequence incoming 47 * data segments. 48 * Andi Kleen: Make sure we never ack data there is not 49 * enough room for. Also make this condition 50 * a fatal error if it might still happen. 51 * Andi Kleen: Add tcp_measure_rcv_mss to make 52 * connections with MSS<min(MTU,ann. MSS) 53 * work without delayed acks. 54 * Andi Kleen: Process packets with PSH set in the 55 * fast path. 56 * J Hadi Salim: ECN support 57 * Andrei Gurtov, 58 * Pasi Sarolahti, 59 * Panu Kuhlberg: Experimental audit of TCP (re)transmission 60 * engine. Lots of bugs are found. 61 * Pasi Sarolahti: F-RTO for dealing with spurious RTOs 62 */ 63 64 #define pr_fmt(fmt) "TCP: " fmt 65 66 #include <linux/mm.h> 67 #include <linux/slab.h> 68 #include <linux/module.h> 69 #include <linux/sysctl.h> 70 #include <linux/kernel.h> 71 #include <net/dst.h> 72 #include <net/tcp.h> 73 #include <net/inet_common.h> 74 #include <linux/ipsec.h> 75 #include <asm/unaligned.h> 76 #include <net/netdma.h> 77 78 int sysctl_tcp_timestamps __read_mostly = 1; 79 int sysctl_tcp_window_scaling __read_mostly = 1; 80 int sysctl_tcp_sack __read_mostly = 1; 81 int sysctl_tcp_fack __read_mostly = 1; 82 int sysctl_tcp_reordering __read_mostly = TCP_FASTRETRANS_THRESH; 83 EXPORT_SYMBOL(sysctl_tcp_reordering); 84 int sysctl_tcp_dsack __read_mostly = 1; 85 int sysctl_tcp_app_win __read_mostly = 31; 86 int sysctl_tcp_adv_win_scale __read_mostly = 1; 87 EXPORT_SYMBOL(sysctl_tcp_adv_win_scale); 88 89 /* rfc5961 challenge ack rate limiting */ 90 int sysctl_tcp_challenge_ack_limit = 100; 91 92 int sysctl_tcp_stdurg __read_mostly; 93 int sysctl_tcp_rfc1337 __read_mostly; 94 int sysctl_tcp_max_orphans __read_mostly = NR_FILE; 95 int sysctl_tcp_frto __read_mostly = 2; 96 97 int sysctl_tcp_thin_dupack __read_mostly; 98 99 int sysctl_tcp_moderate_rcvbuf __read_mostly = 1; 100 int sysctl_tcp_early_retrans __read_mostly = 3; 101 102 #define FLAG_DATA 0x01 /* Incoming frame contained data. */ 103 #define FLAG_WIN_UPDATE 0x02 /* Incoming ACK was a window update. */ 104 #define FLAG_DATA_ACKED 0x04 /* This ACK acknowledged new data. */ 105 #define FLAG_RETRANS_DATA_ACKED 0x08 /* "" "" some of which was retransmitted. */ 106 #define FLAG_SYN_ACKED 0x10 /* This ACK acknowledged SYN. */ 107 #define FLAG_DATA_SACKED 0x20 /* New SACK. */ 108 #define FLAG_ECE 0x40 /* ECE in this ACK */ 109 #define FLAG_SLOWPATH 0x100 /* Do not skip RFC checks for window update.*/ 110 #define FLAG_ORIG_SACK_ACKED 0x200 /* Never retransmitted data are (s)acked */ 111 #define FLAG_SND_UNA_ADVANCED 0x400 /* Snd_una was changed (!= FLAG_DATA_ACKED) */ 112 #define FLAG_DSACKING_ACK 0x800 /* SACK blocks contained D-SACK info */ 113 #define FLAG_SACK_RENEGING 0x2000 /* snd_una advanced to a sacked seq */ 114 #define FLAG_UPDATE_TS_RECENT 0x4000 /* tcp_replace_ts_recent() */ 115 116 #define FLAG_ACKED (FLAG_DATA_ACKED|FLAG_SYN_ACKED) 117 #define FLAG_NOT_DUP (FLAG_DATA|FLAG_WIN_UPDATE|FLAG_ACKED) 118 #define FLAG_CA_ALERT (FLAG_DATA_SACKED|FLAG_ECE) 119 #define FLAG_FORWARD_PROGRESS (FLAG_ACKED|FLAG_DATA_SACKED) 120 121 #define TCP_REMNANT (TCP_FLAG_FIN|TCP_FLAG_URG|TCP_FLAG_SYN|TCP_FLAG_PSH) 122 #define TCP_HP_BITS (~(TCP_RESERVED_BITS|TCP_FLAG_PSH)) 123 124 /* Adapt the MSS value used to make delayed ack decision to the 125 * real world. 126 */ 127 static void tcp_measure_rcv_mss(struct sock *sk, const struct sk_buff *skb) 128 { 129 struct inet_connection_sock *icsk = inet_csk(sk); 130 const unsigned int lss = icsk->icsk_ack.last_seg_size; 131 unsigned int len; 132 133 icsk->icsk_ack.last_seg_size = 0; 134 135 /* skb->len may jitter because of SACKs, even if peer 136 * sends good full-sized frames. 137 */ 138 len = skb_shinfo(skb)->gso_size ? : skb->len; 139 if (len >= icsk->icsk_ack.rcv_mss) { 140 icsk->icsk_ack.rcv_mss = len; 141 } else { 142 /* Otherwise, we make more careful check taking into account, 143 * that SACKs block is variable. 144 * 145 * "len" is invariant segment length, including TCP header. 146 */ 147 len += skb->data - skb_transport_header(skb); 148 if (len >= TCP_MSS_DEFAULT + sizeof(struct tcphdr) || 149 /* If PSH is not set, packet should be 150 * full sized, provided peer TCP is not badly broken. 151 * This observation (if it is correct 8)) allows 152 * to handle super-low mtu links fairly. 153 */ 154 (len >= TCP_MIN_MSS + sizeof(struct tcphdr) && 155 !(tcp_flag_word(tcp_hdr(skb)) & TCP_REMNANT))) { 156 /* Subtract also invariant (if peer is RFC compliant), 157 * tcp header plus fixed timestamp option length. 158 * Resulting "len" is MSS free of SACK jitter. 159 */ 160 len -= tcp_sk(sk)->tcp_header_len; 161 icsk->icsk_ack.last_seg_size = len; 162 if (len == lss) { 163 icsk->icsk_ack.rcv_mss = len; 164 return; 165 } 166 } 167 if (icsk->icsk_ack.pending & ICSK_ACK_PUSHED) 168 icsk->icsk_ack.pending |= ICSK_ACK_PUSHED2; 169 icsk->icsk_ack.pending |= ICSK_ACK_PUSHED; 170 } 171 } 172 173 static void tcp_incr_quickack(struct sock *sk) 174 { 175 struct inet_connection_sock *icsk = inet_csk(sk); 176 unsigned int quickacks = tcp_sk(sk)->rcv_wnd / (2 * icsk->icsk_ack.rcv_mss); 177 178 if (quickacks == 0) 179 quickacks = 2; 180 if (quickacks > icsk->icsk_ack.quick) 181 icsk->icsk_ack.quick = min(quickacks, TCP_MAX_QUICKACKS); 182 } 183 184 static void tcp_enter_quickack_mode(struct sock *sk) 185 { 186 struct inet_connection_sock *icsk = inet_csk(sk); 187 tcp_incr_quickack(sk); 188 icsk->icsk_ack.pingpong = 0; 189 icsk->icsk_ack.ato = TCP_ATO_MIN; 190 } 191 192 /* Send ACKs quickly, if "quick" count is not exhausted 193 * and the session is not interactive. 194 */ 195 196 static inline bool tcp_in_quickack_mode(const struct sock *sk) 197 { 198 const struct inet_connection_sock *icsk = inet_csk(sk); 199 200 return icsk->icsk_ack.quick && !icsk->icsk_ack.pingpong; 201 } 202 203 static inline void TCP_ECN_queue_cwr(struct tcp_sock *tp) 204 { 205 if (tp->ecn_flags & TCP_ECN_OK) 206 tp->ecn_flags |= TCP_ECN_QUEUE_CWR; 207 } 208 209 static inline void TCP_ECN_accept_cwr(struct tcp_sock *tp, const struct sk_buff *skb) 210 { 211 if (tcp_hdr(skb)->cwr) 212 tp->ecn_flags &= ~TCP_ECN_DEMAND_CWR; 213 } 214 215 static inline void TCP_ECN_withdraw_cwr(struct tcp_sock *tp) 216 { 217 tp->ecn_flags &= ~TCP_ECN_DEMAND_CWR; 218 } 219 220 static inline void TCP_ECN_check_ce(struct tcp_sock *tp, const struct sk_buff *skb) 221 { 222 if (!(tp->ecn_flags & TCP_ECN_OK)) 223 return; 224 225 switch (TCP_SKB_CB(skb)->ip_dsfield & INET_ECN_MASK) { 226 case INET_ECN_NOT_ECT: 227 /* Funny extension: if ECT is not set on a segment, 228 * and we already seen ECT on a previous segment, 229 * it is probably a retransmit. 230 */ 231 if (tp->ecn_flags & TCP_ECN_SEEN) 232 tcp_enter_quickack_mode((struct sock *)tp); 233 break; 234 case INET_ECN_CE: 235 if (!(tp->ecn_flags & TCP_ECN_DEMAND_CWR)) { 236 /* Better not delay acks, sender can have a very low cwnd */ 237 tcp_enter_quickack_mode((struct sock *)tp); 238 tp->ecn_flags |= TCP_ECN_DEMAND_CWR; 239 } 240 /* fallinto */ 241 default: 242 tp->ecn_flags |= TCP_ECN_SEEN; 243 } 244 } 245 246 static inline void TCP_ECN_rcv_synack(struct tcp_sock *tp, const struct tcphdr *th) 247 { 248 if ((tp->ecn_flags & TCP_ECN_OK) && (!th->ece || th->cwr)) 249 tp->ecn_flags &= ~TCP_ECN_OK; 250 } 251 252 static inline void TCP_ECN_rcv_syn(struct tcp_sock *tp, const struct tcphdr *th) 253 { 254 if ((tp->ecn_flags & TCP_ECN_OK) && (!th->ece || !th->cwr)) 255 tp->ecn_flags &= ~TCP_ECN_OK; 256 } 257 258 static bool TCP_ECN_rcv_ecn_echo(const struct tcp_sock *tp, const struct tcphdr *th) 259 { 260 if (th->ece && !th->syn && (tp->ecn_flags & TCP_ECN_OK)) 261 return true; 262 return false; 263 } 264 265 /* Buffer size and advertised window tuning. 266 * 267 * 1. Tuning sk->sk_sndbuf, when connection enters established state. 268 */ 269 270 static void tcp_fixup_sndbuf(struct sock *sk) 271 { 272 int sndmem = SKB_TRUESIZE(tcp_sk(sk)->rx_opt.mss_clamp + MAX_TCP_HEADER); 273 274 sndmem *= TCP_INIT_CWND; 275 if (sk->sk_sndbuf < sndmem) 276 sk->sk_sndbuf = min(sndmem, sysctl_tcp_wmem[2]); 277 } 278 279 /* 2. Tuning advertised window (window_clamp, rcv_ssthresh) 280 * 281 * All tcp_full_space() is split to two parts: "network" buffer, allocated 282 * forward and advertised in receiver window (tp->rcv_wnd) and 283 * "application buffer", required to isolate scheduling/application 284 * latencies from network. 285 * window_clamp is maximal advertised window. It can be less than 286 * tcp_full_space(), in this case tcp_full_space() - window_clamp 287 * is reserved for "application" buffer. The less window_clamp is 288 * the smoother our behaviour from viewpoint of network, but the lower 289 * throughput and the higher sensitivity of the connection to losses. 8) 290 * 291 * rcv_ssthresh is more strict window_clamp used at "slow start" 292 * phase to predict further behaviour of this connection. 293 * It is used for two goals: 294 * - to enforce header prediction at sender, even when application 295 * requires some significant "application buffer". It is check #1. 296 * - to prevent pruning of receive queue because of misprediction 297 * of receiver window. Check #2. 298 * 299 * The scheme does not work when sender sends good segments opening 300 * window and then starts to feed us spaghetti. But it should work 301 * in common situations. Otherwise, we have to rely on queue collapsing. 302 */ 303 304 /* Slow part of check#2. */ 305 static int __tcp_grow_window(const struct sock *sk, const struct sk_buff *skb) 306 { 307 struct tcp_sock *tp = tcp_sk(sk); 308 /* Optimize this! */ 309 int truesize = tcp_win_from_space(skb->truesize) >> 1; 310 int window = tcp_win_from_space(sysctl_tcp_rmem[2]) >> 1; 311 312 while (tp->rcv_ssthresh <= window) { 313 if (truesize <= skb->len) 314 return 2 * inet_csk(sk)->icsk_ack.rcv_mss; 315 316 truesize >>= 1; 317 window >>= 1; 318 } 319 return 0; 320 } 321 322 static void tcp_grow_window(struct sock *sk, const struct sk_buff *skb) 323 { 324 struct tcp_sock *tp = tcp_sk(sk); 325 326 /* Check #1 */ 327 if (tp->rcv_ssthresh < tp->window_clamp && 328 (int)tp->rcv_ssthresh < tcp_space(sk) && 329 !sk_under_memory_pressure(sk)) { 330 int incr; 331 332 /* Check #2. Increase window, if skb with such overhead 333 * will fit to rcvbuf in future. 334 */ 335 if (tcp_win_from_space(skb->truesize) <= skb->len) 336 incr = 2 * tp->advmss; 337 else 338 incr = __tcp_grow_window(sk, skb); 339 340 if (incr) { 341 incr = max_t(int, incr, 2 * skb->len); 342 tp->rcv_ssthresh = min(tp->rcv_ssthresh + incr, 343 tp->window_clamp); 344 inet_csk(sk)->icsk_ack.quick |= 1; 345 } 346 } 347 } 348 349 /* 3. Tuning rcvbuf, when connection enters established state. */ 350 static void tcp_fixup_rcvbuf(struct sock *sk) 351 { 352 u32 mss = tcp_sk(sk)->advmss; 353 int rcvmem; 354 355 rcvmem = 2 * SKB_TRUESIZE(mss + MAX_TCP_HEADER) * 356 tcp_default_init_rwnd(mss); 357 358 if (sk->sk_rcvbuf < rcvmem) 359 sk->sk_rcvbuf = min(rcvmem, sysctl_tcp_rmem[2]); 360 } 361 362 /* 4. Try to fixup all. It is made immediately after connection enters 363 * established state. 364 */ 365 void tcp_init_buffer_space(struct sock *sk) 366 { 367 struct tcp_sock *tp = tcp_sk(sk); 368 int maxwin; 369 370 if (!(sk->sk_userlocks & SOCK_RCVBUF_LOCK)) 371 tcp_fixup_rcvbuf(sk); 372 if (!(sk->sk_userlocks & SOCK_SNDBUF_LOCK)) 373 tcp_fixup_sndbuf(sk); 374 375 tp->rcvq_space.space = tp->rcv_wnd; 376 377 maxwin = tcp_full_space(sk); 378 379 if (tp->window_clamp >= maxwin) { 380 tp->window_clamp = maxwin; 381 382 if (sysctl_tcp_app_win && maxwin > 4 * tp->advmss) 383 tp->window_clamp = max(maxwin - 384 (maxwin >> sysctl_tcp_app_win), 385 4 * tp->advmss); 386 } 387 388 /* Force reservation of one segment. */ 389 if (sysctl_tcp_app_win && 390 tp->window_clamp > 2 * tp->advmss && 391 tp->window_clamp + tp->advmss > maxwin) 392 tp->window_clamp = max(2 * tp->advmss, maxwin - tp->advmss); 393 394 tp->rcv_ssthresh = min(tp->rcv_ssthresh, tp->window_clamp); 395 tp->snd_cwnd_stamp = tcp_time_stamp; 396 } 397 398 /* 5. Recalculate window clamp after socket hit its memory bounds. */ 399 static void tcp_clamp_window(struct sock *sk) 400 { 401 struct tcp_sock *tp = tcp_sk(sk); 402 struct inet_connection_sock *icsk = inet_csk(sk); 403 404 icsk->icsk_ack.quick = 0; 405 406 if (sk->sk_rcvbuf < sysctl_tcp_rmem[2] && 407 !(sk->sk_userlocks & SOCK_RCVBUF_LOCK) && 408 !sk_under_memory_pressure(sk) && 409 sk_memory_allocated(sk) < sk_prot_mem_limits(sk, 0)) { 410 sk->sk_rcvbuf = min(atomic_read(&sk->sk_rmem_alloc), 411 sysctl_tcp_rmem[2]); 412 } 413 if (atomic_read(&sk->sk_rmem_alloc) > sk->sk_rcvbuf) 414 tp->rcv_ssthresh = min(tp->window_clamp, 2U * tp->advmss); 415 } 416 417 /* Initialize RCV_MSS value. 418 * RCV_MSS is an our guess about MSS used by the peer. 419 * We haven't any direct information about the MSS. 420 * It's better to underestimate the RCV_MSS rather than overestimate. 421 * Overestimations make us ACKing less frequently than needed. 422 * Underestimations are more easy to detect and fix by tcp_measure_rcv_mss(). 423 */ 424 void tcp_initialize_rcv_mss(struct sock *sk) 425 { 426 const struct tcp_sock *tp = tcp_sk(sk); 427 unsigned int hint = min_t(unsigned int, tp->advmss, tp->mss_cache); 428 429 hint = min(hint, tp->rcv_wnd / 2); 430 hint = min(hint, TCP_MSS_DEFAULT); 431 hint = max(hint, TCP_MIN_MSS); 432 433 inet_csk(sk)->icsk_ack.rcv_mss = hint; 434 } 435 EXPORT_SYMBOL(tcp_initialize_rcv_mss); 436 437 /* Receiver "autotuning" code. 438 * 439 * The algorithm for RTT estimation w/o timestamps is based on 440 * Dynamic Right-Sizing (DRS) by Wu Feng and Mike Fisk of LANL. 441 * <http://public.lanl.gov/radiant/pubs.html#DRS> 442 * 443 * More detail on this code can be found at 444 * <http://staff.psc.edu/jheffner/>, 445 * though this reference is out of date. A new paper 446 * is pending. 447 */ 448 static void tcp_rcv_rtt_update(struct tcp_sock *tp, u32 sample, int win_dep) 449 { 450 u32 new_sample = tp->rcv_rtt_est.rtt; 451 long m = sample; 452 453 if (m == 0) 454 m = 1; 455 456 if (new_sample != 0) { 457 /* If we sample in larger samples in the non-timestamp 458 * case, we could grossly overestimate the RTT especially 459 * with chatty applications or bulk transfer apps which 460 * are stalled on filesystem I/O. 461 * 462 * Also, since we are only going for a minimum in the 463 * non-timestamp case, we do not smooth things out 464 * else with timestamps disabled convergence takes too 465 * long. 466 */ 467 if (!win_dep) { 468 m -= (new_sample >> 3); 469 new_sample += m; 470 } else { 471 m <<= 3; 472 if (m < new_sample) 473 new_sample = m; 474 } 475 } else { 476 /* No previous measure. */ 477 new_sample = m << 3; 478 } 479 480 if (tp->rcv_rtt_est.rtt != new_sample) 481 tp->rcv_rtt_est.rtt = new_sample; 482 } 483 484 static inline void tcp_rcv_rtt_measure(struct tcp_sock *tp) 485 { 486 if (tp->rcv_rtt_est.time == 0) 487 goto new_measure; 488 if (before(tp->rcv_nxt, tp->rcv_rtt_est.seq)) 489 return; 490 tcp_rcv_rtt_update(tp, tcp_time_stamp - tp->rcv_rtt_est.time, 1); 491 492 new_measure: 493 tp->rcv_rtt_est.seq = tp->rcv_nxt + tp->rcv_wnd; 494 tp->rcv_rtt_est.time = tcp_time_stamp; 495 } 496 497 static inline void tcp_rcv_rtt_measure_ts(struct sock *sk, 498 const struct sk_buff *skb) 499 { 500 struct tcp_sock *tp = tcp_sk(sk); 501 if (tp->rx_opt.rcv_tsecr && 502 (TCP_SKB_CB(skb)->end_seq - 503 TCP_SKB_CB(skb)->seq >= inet_csk(sk)->icsk_ack.rcv_mss)) 504 tcp_rcv_rtt_update(tp, tcp_time_stamp - tp->rx_opt.rcv_tsecr, 0); 505 } 506 507 /* 508 * This function should be called every time data is copied to user space. 509 * It calculates the appropriate TCP receive buffer space. 510 */ 511 void tcp_rcv_space_adjust(struct sock *sk) 512 { 513 struct tcp_sock *tp = tcp_sk(sk); 514 int time; 515 int space; 516 517 if (tp->rcvq_space.time == 0) 518 goto new_measure; 519 520 time = tcp_time_stamp - tp->rcvq_space.time; 521 if (time < (tp->rcv_rtt_est.rtt >> 3) || tp->rcv_rtt_est.rtt == 0) 522 return; 523 524 space = 2 * (tp->copied_seq - tp->rcvq_space.seq); 525 526 space = max(tp->rcvq_space.space, space); 527 528 if (tp->rcvq_space.space != space) { 529 int rcvmem; 530 531 tp->rcvq_space.space = space; 532 533 if (sysctl_tcp_moderate_rcvbuf && 534 !(sk->sk_userlocks & SOCK_RCVBUF_LOCK)) { 535 int new_clamp = space; 536 537 /* Receive space grows, normalize in order to 538 * take into account packet headers and sk_buff 539 * structure overhead. 540 */ 541 space /= tp->advmss; 542 if (!space) 543 space = 1; 544 rcvmem = SKB_TRUESIZE(tp->advmss + MAX_TCP_HEADER); 545 while (tcp_win_from_space(rcvmem) < tp->advmss) 546 rcvmem += 128; 547 space *= rcvmem; 548 space = min(space, sysctl_tcp_rmem[2]); 549 if (space > sk->sk_rcvbuf) { 550 sk->sk_rcvbuf = space; 551 552 /* Make the window clamp follow along. */ 553 tp->window_clamp = new_clamp; 554 } 555 } 556 } 557 558 new_measure: 559 tp->rcvq_space.seq = tp->copied_seq; 560 tp->rcvq_space.time = tcp_time_stamp; 561 } 562 563 /* There is something which you must keep in mind when you analyze the 564 * behavior of the tp->ato delayed ack timeout interval. When a 565 * connection starts up, we want to ack as quickly as possible. The 566 * problem is that "good" TCP's do slow start at the beginning of data 567 * transmission. The means that until we send the first few ACK's the 568 * sender will sit on his end and only queue most of his data, because 569 * he can only send snd_cwnd unacked packets at any given time. For 570 * each ACK we send, he increments snd_cwnd and transmits more of his 571 * queue. -DaveM 572 */ 573 static void tcp_event_data_recv(struct sock *sk, struct sk_buff *skb) 574 { 575 struct tcp_sock *tp = tcp_sk(sk); 576 struct inet_connection_sock *icsk = inet_csk(sk); 577 u32 now; 578 579 inet_csk_schedule_ack(sk); 580 581 tcp_measure_rcv_mss(sk, skb); 582 583 tcp_rcv_rtt_measure(tp); 584 585 now = tcp_time_stamp; 586 587 if (!icsk->icsk_ack.ato) { 588 /* The _first_ data packet received, initialize 589 * delayed ACK engine. 590 */ 591 tcp_incr_quickack(sk); 592 icsk->icsk_ack.ato = TCP_ATO_MIN; 593 } else { 594 int m = now - icsk->icsk_ack.lrcvtime; 595 596 if (m <= TCP_ATO_MIN / 2) { 597 /* The fastest case is the first. */ 598 icsk->icsk_ack.ato = (icsk->icsk_ack.ato >> 1) + TCP_ATO_MIN / 2; 599 } else if (m < icsk->icsk_ack.ato) { 600 icsk->icsk_ack.ato = (icsk->icsk_ack.ato >> 1) + m; 601 if (icsk->icsk_ack.ato > icsk->icsk_rto) 602 icsk->icsk_ack.ato = icsk->icsk_rto; 603 } else if (m > icsk->icsk_rto) { 604 /* Too long gap. Apparently sender failed to 605 * restart window, so that we send ACKs quickly. 606 */ 607 tcp_incr_quickack(sk); 608 sk_mem_reclaim(sk); 609 } 610 } 611 icsk->icsk_ack.lrcvtime = now; 612 613 TCP_ECN_check_ce(tp, skb); 614 615 if (skb->len >= 128) 616 tcp_grow_window(sk, skb); 617 } 618 619 /* Called to compute a smoothed rtt estimate. The data fed to this 620 * routine either comes from timestamps, or from segments that were 621 * known _not_ to have been retransmitted [see Karn/Partridge 622 * Proceedings SIGCOMM 87]. The algorithm is from the SIGCOMM 88 623 * piece by Van Jacobson. 624 * NOTE: the next three routines used to be one big routine. 625 * To save cycles in the RFC 1323 implementation it was better to break 626 * it up into three procedures. -- erics 627 */ 628 static void tcp_rtt_estimator(struct sock *sk, const __u32 mrtt) 629 { 630 struct tcp_sock *tp = tcp_sk(sk); 631 long m = mrtt; /* RTT */ 632 633 /* The following amusing code comes from Jacobson's 634 * article in SIGCOMM '88. Note that rtt and mdev 635 * are scaled versions of rtt and mean deviation. 636 * This is designed to be as fast as possible 637 * m stands for "measurement". 638 * 639 * On a 1990 paper the rto value is changed to: 640 * RTO = rtt + 4 * mdev 641 * 642 * Funny. This algorithm seems to be very broken. 643 * These formulae increase RTO, when it should be decreased, increase 644 * too slowly, when it should be increased quickly, decrease too quickly 645 * etc. I guess in BSD RTO takes ONE value, so that it is absolutely 646 * does not matter how to _calculate_ it. Seems, it was trap 647 * that VJ failed to avoid. 8) 648 */ 649 if (m == 0) 650 m = 1; 651 if (tp->srtt != 0) { 652 m -= (tp->srtt >> 3); /* m is now error in rtt est */ 653 tp->srtt += m; /* rtt = 7/8 rtt + 1/8 new */ 654 if (m < 0) { 655 m = -m; /* m is now abs(error) */ 656 m -= (tp->mdev >> 2); /* similar update on mdev */ 657 /* This is similar to one of Eifel findings. 658 * Eifel blocks mdev updates when rtt decreases. 659 * This solution is a bit different: we use finer gain 660 * for mdev in this case (alpha*beta). 661 * Like Eifel it also prevents growth of rto, 662 * but also it limits too fast rto decreases, 663 * happening in pure Eifel. 664 */ 665 if (m > 0) 666 m >>= 3; 667 } else { 668 m -= (tp->mdev >> 2); /* similar update on mdev */ 669 } 670 tp->mdev += m; /* mdev = 3/4 mdev + 1/4 new */ 671 if (tp->mdev > tp->mdev_max) { 672 tp->mdev_max = tp->mdev; 673 if (tp->mdev_max > tp->rttvar) 674 tp->rttvar = tp->mdev_max; 675 } 676 if (after(tp->snd_una, tp->rtt_seq)) { 677 if (tp->mdev_max < tp->rttvar) 678 tp->rttvar -= (tp->rttvar - tp->mdev_max) >> 2; 679 tp->rtt_seq = tp->snd_nxt; 680 tp->mdev_max = tcp_rto_min(sk); 681 } 682 } else { 683 /* no previous measure. */ 684 tp->srtt = m << 3; /* take the measured time to be rtt */ 685 tp->mdev = m << 1; /* make sure rto = 3*rtt */ 686 tp->mdev_max = tp->rttvar = max(tp->mdev, tcp_rto_min(sk)); 687 tp->rtt_seq = tp->snd_nxt; 688 } 689 } 690 691 /* Set the sk_pacing_rate to allow proper sizing of TSO packets. 692 * Note: TCP stack does not yet implement pacing. 693 * FQ packet scheduler can be used to implement cheap but effective 694 * TCP pacing, to smooth the burst on large writes when packets 695 * in flight is significantly lower than cwnd (or rwin) 696 */ 697 static void tcp_update_pacing_rate(struct sock *sk) 698 { 699 const struct tcp_sock *tp = tcp_sk(sk); 700 u64 rate; 701 702 /* set sk_pacing_rate to 200 % of current rate (mss * cwnd / srtt) */ 703 rate = (u64)tp->mss_cache * 2 * (HZ << 3); 704 705 rate *= max(tp->snd_cwnd, tp->packets_out); 706 707 /* Correction for small srtt : minimum srtt being 8 (1 jiffy << 3), 708 * be conservative and assume srtt = 1 (125 us instead of 1.25 ms) 709 * We probably need usec resolution in the future. 710 * Note: This also takes care of possible srtt=0 case, 711 * when tcp_rtt_estimator() was not yet called. 712 */ 713 if (tp->srtt > 8 + 2) 714 do_div(rate, tp->srtt); 715 716 sk->sk_pacing_rate = min_t(u64, rate, ~0U); 717 } 718 719 /* Calculate rto without backoff. This is the second half of Van Jacobson's 720 * routine referred to above. 721 */ 722 void tcp_set_rto(struct sock *sk) 723 { 724 const struct tcp_sock *tp = tcp_sk(sk); 725 /* Old crap is replaced with new one. 8) 726 * 727 * More seriously: 728 * 1. If rtt variance happened to be less 50msec, it is hallucination. 729 * It cannot be less due to utterly erratic ACK generation made 730 * at least by solaris and freebsd. "Erratic ACKs" has _nothing_ 731 * to do with delayed acks, because at cwnd>2 true delack timeout 732 * is invisible. Actually, Linux-2.4 also generates erratic 733 * ACKs in some circumstances. 734 */ 735 inet_csk(sk)->icsk_rto = __tcp_set_rto(tp); 736 737 /* 2. Fixups made earlier cannot be right. 738 * If we do not estimate RTO correctly without them, 739 * all the algo is pure shit and should be replaced 740 * with correct one. It is exactly, which we pretend to do. 741 */ 742 743 /* NOTE: clamping at TCP_RTO_MIN is not required, current algo 744 * guarantees that rto is higher. 745 */ 746 tcp_bound_rto(sk); 747 } 748 749 __u32 tcp_init_cwnd(const struct tcp_sock *tp, const struct dst_entry *dst) 750 { 751 __u32 cwnd = (dst ? dst_metric(dst, RTAX_INITCWND) : 0); 752 753 if (!cwnd) 754 cwnd = TCP_INIT_CWND; 755 return min_t(__u32, cwnd, tp->snd_cwnd_clamp); 756 } 757 758 /* 759 * Packet counting of FACK is based on in-order assumptions, therefore TCP 760 * disables it when reordering is detected 761 */ 762 void tcp_disable_fack(struct tcp_sock *tp) 763 { 764 /* RFC3517 uses different metric in lost marker => reset on change */ 765 if (tcp_is_fack(tp)) 766 tp->lost_skb_hint = NULL; 767 tp->rx_opt.sack_ok &= ~TCP_FACK_ENABLED; 768 } 769 770 /* Take a notice that peer is sending D-SACKs */ 771 static void tcp_dsack_seen(struct tcp_sock *tp) 772 { 773 tp->rx_opt.sack_ok |= TCP_DSACK_SEEN; 774 } 775 776 static void tcp_update_reordering(struct sock *sk, const int metric, 777 const int ts) 778 { 779 struct tcp_sock *tp = tcp_sk(sk); 780 if (metric > tp->reordering) { 781 int mib_idx; 782 783 tp->reordering = min(TCP_MAX_REORDERING, metric); 784 785 /* This exciting event is worth to be remembered. 8) */ 786 if (ts) 787 mib_idx = LINUX_MIB_TCPTSREORDER; 788 else if (tcp_is_reno(tp)) 789 mib_idx = LINUX_MIB_TCPRENOREORDER; 790 else if (tcp_is_fack(tp)) 791 mib_idx = LINUX_MIB_TCPFACKREORDER; 792 else 793 mib_idx = LINUX_MIB_TCPSACKREORDER; 794 795 NET_INC_STATS_BH(sock_net(sk), mib_idx); 796 #if FASTRETRANS_DEBUG > 1 797 pr_debug("Disorder%d %d %u f%u s%u rr%d\n", 798 tp->rx_opt.sack_ok, inet_csk(sk)->icsk_ca_state, 799 tp->reordering, 800 tp->fackets_out, 801 tp->sacked_out, 802 tp->undo_marker ? tp->undo_retrans : 0); 803 #endif 804 tcp_disable_fack(tp); 805 } 806 807 if (metric > 0) 808 tcp_disable_early_retrans(tp); 809 } 810 811 /* This must be called before lost_out is incremented */ 812 static void tcp_verify_retransmit_hint(struct tcp_sock *tp, struct sk_buff *skb) 813 { 814 if ((tp->retransmit_skb_hint == NULL) || 815 before(TCP_SKB_CB(skb)->seq, 816 TCP_SKB_CB(tp->retransmit_skb_hint)->seq)) 817 tp->retransmit_skb_hint = skb; 818 819 if (!tp->lost_out || 820 after(TCP_SKB_CB(skb)->end_seq, tp->retransmit_high)) 821 tp->retransmit_high = TCP_SKB_CB(skb)->end_seq; 822 } 823 824 static void tcp_skb_mark_lost(struct tcp_sock *tp, struct sk_buff *skb) 825 { 826 if (!(TCP_SKB_CB(skb)->sacked & (TCPCB_LOST|TCPCB_SACKED_ACKED))) { 827 tcp_verify_retransmit_hint(tp, skb); 828 829 tp->lost_out += tcp_skb_pcount(skb); 830 TCP_SKB_CB(skb)->sacked |= TCPCB_LOST; 831 } 832 } 833 834 static void tcp_skb_mark_lost_uncond_verify(struct tcp_sock *tp, 835 struct sk_buff *skb) 836 { 837 tcp_verify_retransmit_hint(tp, skb); 838 839 if (!(TCP_SKB_CB(skb)->sacked & (TCPCB_LOST|TCPCB_SACKED_ACKED))) { 840 tp->lost_out += tcp_skb_pcount(skb); 841 TCP_SKB_CB(skb)->sacked |= TCPCB_LOST; 842 } 843 } 844 845 /* This procedure tags the retransmission queue when SACKs arrive. 846 * 847 * We have three tag bits: SACKED(S), RETRANS(R) and LOST(L). 848 * Packets in queue with these bits set are counted in variables 849 * sacked_out, retrans_out and lost_out, correspondingly. 850 * 851 * Valid combinations are: 852 * Tag InFlight Description 853 * 0 1 - orig segment is in flight. 854 * S 0 - nothing flies, orig reached receiver. 855 * L 0 - nothing flies, orig lost by net. 856 * R 2 - both orig and retransmit are in flight. 857 * L|R 1 - orig is lost, retransmit is in flight. 858 * S|R 1 - orig reached receiver, retrans is still in flight. 859 * (L|S|R is logically valid, it could occur when L|R is sacked, 860 * but it is equivalent to plain S and code short-curcuits it to S. 861 * L|S is logically invalid, it would mean -1 packet in flight 8)) 862 * 863 * These 6 states form finite state machine, controlled by the following events: 864 * 1. New ACK (+SACK) arrives. (tcp_sacktag_write_queue()) 865 * 2. Retransmission. (tcp_retransmit_skb(), tcp_xmit_retransmit_queue()) 866 * 3. Loss detection event of two flavors: 867 * A. Scoreboard estimator decided the packet is lost. 868 * A'. Reno "three dupacks" marks head of queue lost. 869 * A''. Its FACK modification, head until snd.fack is lost. 870 * B. SACK arrives sacking SND.NXT at the moment, when the 871 * segment was retransmitted. 872 * 4. D-SACK added new rule: D-SACK changes any tag to S. 873 * 874 * It is pleasant to note, that state diagram turns out to be commutative, 875 * so that we are allowed not to be bothered by order of our actions, 876 * when multiple events arrive simultaneously. (see the function below). 877 * 878 * Reordering detection. 879 * -------------------- 880 * Reordering metric is maximal distance, which a packet can be displaced 881 * in packet stream. With SACKs we can estimate it: 882 * 883 * 1. SACK fills old hole and the corresponding segment was not 884 * ever retransmitted -> reordering. Alas, we cannot use it 885 * when segment was retransmitted. 886 * 2. The last flaw is solved with D-SACK. D-SACK arrives 887 * for retransmitted and already SACKed segment -> reordering.. 888 * Both of these heuristics are not used in Loss state, when we cannot 889 * account for retransmits accurately. 890 * 891 * SACK block validation. 892 * ---------------------- 893 * 894 * SACK block range validation checks that the received SACK block fits to 895 * the expected sequence limits, i.e., it is between SND.UNA and SND.NXT. 896 * Note that SND.UNA is not included to the range though being valid because 897 * it means that the receiver is rather inconsistent with itself reporting 898 * SACK reneging when it should advance SND.UNA. Such SACK block this is 899 * perfectly valid, however, in light of RFC2018 which explicitly states 900 * that "SACK block MUST reflect the newest segment. Even if the newest 901 * segment is going to be discarded ...", not that it looks very clever 902 * in case of head skb. Due to potentional receiver driven attacks, we 903 * choose to avoid immediate execution of a walk in write queue due to 904 * reneging and defer head skb's loss recovery to standard loss recovery 905 * procedure that will eventually trigger (nothing forbids us doing this). 906 * 907 * Implements also blockage to start_seq wrap-around. Problem lies in the 908 * fact that though start_seq (s) is before end_seq (i.e., not reversed), 909 * there's no guarantee that it will be before snd_nxt (n). The problem 910 * happens when start_seq resides between end_seq wrap (e_w) and snd_nxt 911 * wrap (s_w): 912 * 913 * <- outs wnd -> <- wrapzone -> 914 * u e n u_w e_w s n_w 915 * | | | | | | | 916 * |<------------+------+----- TCP seqno space --------------+---------->| 917 * ...-- <2^31 ->| |<--------... 918 * ...---- >2^31 ------>| |<--------... 919 * 920 * Current code wouldn't be vulnerable but it's better still to discard such 921 * crazy SACK blocks. Doing this check for start_seq alone closes somewhat 922 * similar case (end_seq after snd_nxt wrap) as earlier reversed check in 923 * snd_nxt wrap -> snd_una region will then become "well defined", i.e., 924 * equal to the ideal case (infinite seqno space without wrap caused issues). 925 * 926 * With D-SACK the lower bound is extended to cover sequence space below 927 * SND.UNA down to undo_marker, which is the last point of interest. Yet 928 * again, D-SACK block must not to go across snd_una (for the same reason as 929 * for the normal SACK blocks, explained above). But there all simplicity 930 * ends, TCP might receive valid D-SACKs below that. As long as they reside 931 * fully below undo_marker they do not affect behavior in anyway and can 932 * therefore be safely ignored. In rare cases (which are more or less 933 * theoretical ones), the D-SACK will nicely cross that boundary due to skb 934 * fragmentation and packet reordering past skb's retransmission. To consider 935 * them correctly, the acceptable range must be extended even more though 936 * the exact amount is rather hard to quantify. However, tp->max_window can 937 * be used as an exaggerated estimate. 938 */ 939 static bool tcp_is_sackblock_valid(struct tcp_sock *tp, bool is_dsack, 940 u32 start_seq, u32 end_seq) 941 { 942 /* Too far in future, or reversed (interpretation is ambiguous) */ 943 if (after(end_seq, tp->snd_nxt) || !before(start_seq, end_seq)) 944 return false; 945 946 /* Nasty start_seq wrap-around check (see comments above) */ 947 if (!before(start_seq, tp->snd_nxt)) 948 return false; 949 950 /* In outstanding window? ...This is valid exit for D-SACKs too. 951 * start_seq == snd_una is non-sensical (see comments above) 952 */ 953 if (after(start_seq, tp->snd_una)) 954 return true; 955 956 if (!is_dsack || !tp->undo_marker) 957 return false; 958 959 /* ...Then it's D-SACK, and must reside below snd_una completely */ 960 if (after(end_seq, tp->snd_una)) 961 return false; 962 963 if (!before(start_seq, tp->undo_marker)) 964 return true; 965 966 /* Too old */ 967 if (!after(end_seq, tp->undo_marker)) 968 return false; 969 970 /* Undo_marker boundary crossing (overestimates a lot). Known already: 971 * start_seq < undo_marker and end_seq >= undo_marker. 972 */ 973 return !before(start_seq, end_seq - tp->max_window); 974 } 975 976 /* Check for lost retransmit. This superb idea is borrowed from "ratehalving". 977 * Event "B". Later note: FACK people cheated me again 8), we have to account 978 * for reordering! Ugly, but should help. 979 * 980 * Search retransmitted skbs from write_queue that were sent when snd_nxt was 981 * less than what is now known to be received by the other end (derived from 982 * highest SACK block). Also calculate the lowest snd_nxt among the remaining 983 * retransmitted skbs to avoid some costly processing per ACKs. 984 */ 985 static void tcp_mark_lost_retrans(struct sock *sk) 986 { 987 const struct inet_connection_sock *icsk = inet_csk(sk); 988 struct tcp_sock *tp = tcp_sk(sk); 989 struct sk_buff *skb; 990 int cnt = 0; 991 u32 new_low_seq = tp->snd_nxt; 992 u32 received_upto = tcp_highest_sack_seq(tp); 993 994 if (!tcp_is_fack(tp) || !tp->retrans_out || 995 !after(received_upto, tp->lost_retrans_low) || 996 icsk->icsk_ca_state != TCP_CA_Recovery) 997 return; 998 999 tcp_for_write_queue(skb, sk) { 1000 u32 ack_seq = TCP_SKB_CB(skb)->ack_seq; 1001 1002 if (skb == tcp_send_head(sk)) 1003 break; 1004 if (cnt == tp->retrans_out) 1005 break; 1006 if (!after(TCP_SKB_CB(skb)->end_seq, tp->snd_una)) 1007 continue; 1008 1009 if (!(TCP_SKB_CB(skb)->sacked & TCPCB_SACKED_RETRANS)) 1010 continue; 1011 1012 /* TODO: We would like to get rid of tcp_is_fack(tp) only 1013 * constraint here (see above) but figuring out that at 1014 * least tp->reordering SACK blocks reside between ack_seq 1015 * and received_upto is not easy task to do cheaply with 1016 * the available datastructures. 1017 * 1018 * Whether FACK should check here for tp->reordering segs 1019 * in-between one could argue for either way (it would be 1020 * rather simple to implement as we could count fack_count 1021 * during the walk and do tp->fackets_out - fack_count). 1022 */ 1023 if (after(received_upto, ack_seq)) { 1024 TCP_SKB_CB(skb)->sacked &= ~TCPCB_SACKED_RETRANS; 1025 tp->retrans_out -= tcp_skb_pcount(skb); 1026 1027 tcp_skb_mark_lost_uncond_verify(tp, skb); 1028 NET_INC_STATS_BH(sock_net(sk), LINUX_MIB_TCPLOSTRETRANSMIT); 1029 } else { 1030 if (before(ack_seq, new_low_seq)) 1031 new_low_seq = ack_seq; 1032 cnt += tcp_skb_pcount(skb); 1033 } 1034 } 1035 1036 if (tp->retrans_out) 1037 tp->lost_retrans_low = new_low_seq; 1038 } 1039 1040 static bool tcp_check_dsack(struct sock *sk, const struct sk_buff *ack_skb, 1041 struct tcp_sack_block_wire *sp, int num_sacks, 1042 u32 prior_snd_una) 1043 { 1044 struct tcp_sock *tp = tcp_sk(sk); 1045 u32 start_seq_0 = get_unaligned_be32(&sp[0].start_seq); 1046 u32 end_seq_0 = get_unaligned_be32(&sp[0].end_seq); 1047 bool dup_sack = false; 1048 1049 if (before(start_seq_0, TCP_SKB_CB(ack_skb)->ack_seq)) { 1050 dup_sack = true; 1051 tcp_dsack_seen(tp); 1052 NET_INC_STATS_BH(sock_net(sk), LINUX_MIB_TCPDSACKRECV); 1053 } else if (num_sacks > 1) { 1054 u32 end_seq_1 = get_unaligned_be32(&sp[1].end_seq); 1055 u32 start_seq_1 = get_unaligned_be32(&sp[1].start_seq); 1056 1057 if (!after(end_seq_0, end_seq_1) && 1058 !before(start_seq_0, start_seq_1)) { 1059 dup_sack = true; 1060 tcp_dsack_seen(tp); 1061 NET_INC_STATS_BH(sock_net(sk), 1062 LINUX_MIB_TCPDSACKOFORECV); 1063 } 1064 } 1065 1066 /* D-SACK for already forgotten data... Do dumb counting. */ 1067 if (dup_sack && tp->undo_marker && tp->undo_retrans && 1068 !after(end_seq_0, prior_snd_una) && 1069 after(end_seq_0, tp->undo_marker)) 1070 tp->undo_retrans--; 1071 1072 return dup_sack; 1073 } 1074 1075 struct tcp_sacktag_state { 1076 int reord; 1077 int fack_count; 1078 int flag; 1079 s32 rtt; /* RTT measured by SACKing never-retransmitted data */ 1080 }; 1081 1082 /* Check if skb is fully within the SACK block. In presence of GSO skbs, 1083 * the incoming SACK may not exactly match but we can find smaller MSS 1084 * aligned portion of it that matches. Therefore we might need to fragment 1085 * which may fail and creates some hassle (caller must handle error case 1086 * returns). 1087 * 1088 * FIXME: this could be merged to shift decision code 1089 */ 1090 static int tcp_match_skb_to_sack(struct sock *sk, struct sk_buff *skb, 1091 u32 start_seq, u32 end_seq) 1092 { 1093 int err; 1094 bool in_sack; 1095 unsigned int pkt_len; 1096 unsigned int mss; 1097 1098 in_sack = !after(start_seq, TCP_SKB_CB(skb)->seq) && 1099 !before(end_seq, TCP_SKB_CB(skb)->end_seq); 1100 1101 if (tcp_skb_pcount(skb) > 1 && !in_sack && 1102 after(TCP_SKB_CB(skb)->end_seq, start_seq)) { 1103 mss = tcp_skb_mss(skb); 1104 in_sack = !after(start_seq, TCP_SKB_CB(skb)->seq); 1105 1106 if (!in_sack) { 1107 pkt_len = start_seq - TCP_SKB_CB(skb)->seq; 1108 if (pkt_len < mss) 1109 pkt_len = mss; 1110 } else { 1111 pkt_len = end_seq - TCP_SKB_CB(skb)->seq; 1112 if (pkt_len < mss) 1113 return -EINVAL; 1114 } 1115 1116 /* Round if necessary so that SACKs cover only full MSSes 1117 * and/or the remaining small portion (if present) 1118 */ 1119 if (pkt_len > mss) { 1120 unsigned int new_len = (pkt_len / mss) * mss; 1121 if (!in_sack && new_len < pkt_len) { 1122 new_len += mss; 1123 if (new_len > skb->len) 1124 return 0; 1125 } 1126 pkt_len = new_len; 1127 } 1128 err = tcp_fragment(sk, skb, pkt_len, mss); 1129 if (err < 0) 1130 return err; 1131 } 1132 1133 return in_sack; 1134 } 1135 1136 /* Mark the given newly-SACKed range as such, adjusting counters and hints. */ 1137 static u8 tcp_sacktag_one(struct sock *sk, 1138 struct tcp_sacktag_state *state, u8 sacked, 1139 u32 start_seq, u32 end_seq, 1140 int dup_sack, int pcount, u32 xmit_time) 1141 { 1142 struct tcp_sock *tp = tcp_sk(sk); 1143 int fack_count = state->fack_count; 1144 1145 /* Account D-SACK for retransmitted packet. */ 1146 if (dup_sack && (sacked & TCPCB_RETRANS)) { 1147 if (tp->undo_marker && tp->undo_retrans && 1148 after(end_seq, tp->undo_marker)) 1149 tp->undo_retrans--; 1150 if (sacked & TCPCB_SACKED_ACKED) 1151 state->reord = min(fack_count, state->reord); 1152 } 1153 1154 /* Nothing to do; acked frame is about to be dropped (was ACKed). */ 1155 if (!after(end_seq, tp->snd_una)) 1156 return sacked; 1157 1158 if (!(sacked & TCPCB_SACKED_ACKED)) { 1159 if (sacked & TCPCB_SACKED_RETRANS) { 1160 /* If the segment is not tagged as lost, 1161 * we do not clear RETRANS, believing 1162 * that retransmission is still in flight. 1163 */ 1164 if (sacked & TCPCB_LOST) { 1165 sacked &= ~(TCPCB_LOST|TCPCB_SACKED_RETRANS); 1166 tp->lost_out -= pcount; 1167 tp->retrans_out -= pcount; 1168 } 1169 } else { 1170 if (!(sacked & TCPCB_RETRANS)) { 1171 /* New sack for not retransmitted frame, 1172 * which was in hole. It is reordering. 1173 */ 1174 if (before(start_seq, 1175 tcp_highest_sack_seq(tp))) 1176 state->reord = min(fack_count, 1177 state->reord); 1178 if (!after(end_seq, tp->high_seq)) 1179 state->flag |= FLAG_ORIG_SACK_ACKED; 1180 /* Pick the earliest sequence sacked for RTT */ 1181 if (state->rtt < 0) 1182 state->rtt = tcp_time_stamp - xmit_time; 1183 } 1184 1185 if (sacked & TCPCB_LOST) { 1186 sacked &= ~TCPCB_LOST; 1187 tp->lost_out -= pcount; 1188 } 1189 } 1190 1191 sacked |= TCPCB_SACKED_ACKED; 1192 state->flag |= FLAG_DATA_SACKED; 1193 tp->sacked_out += pcount; 1194 1195 fack_count += pcount; 1196 1197 /* Lost marker hint past SACKed? Tweak RFC3517 cnt */ 1198 if (!tcp_is_fack(tp) && (tp->lost_skb_hint != NULL) && 1199 before(start_seq, TCP_SKB_CB(tp->lost_skb_hint)->seq)) 1200 tp->lost_cnt_hint += pcount; 1201 1202 if (fack_count > tp->fackets_out) 1203 tp->fackets_out = fack_count; 1204 } 1205 1206 /* D-SACK. We can detect redundant retransmission in S|R and plain R 1207 * frames and clear it. undo_retrans is decreased above, L|R frames 1208 * are accounted above as well. 1209 */ 1210 if (dup_sack && (sacked & TCPCB_SACKED_RETRANS)) { 1211 sacked &= ~TCPCB_SACKED_RETRANS; 1212 tp->retrans_out -= pcount; 1213 } 1214 1215 return sacked; 1216 } 1217 1218 /* Shift newly-SACKed bytes from this skb to the immediately previous 1219 * already-SACKed sk_buff. Mark the newly-SACKed bytes as such. 1220 */ 1221 static bool tcp_shifted_skb(struct sock *sk, struct sk_buff *skb, 1222 struct tcp_sacktag_state *state, 1223 unsigned int pcount, int shifted, int mss, 1224 bool dup_sack) 1225 { 1226 struct tcp_sock *tp = tcp_sk(sk); 1227 struct sk_buff *prev = tcp_write_queue_prev(sk, skb); 1228 u32 start_seq = TCP_SKB_CB(skb)->seq; /* start of newly-SACKed */ 1229 u32 end_seq = start_seq + shifted; /* end of newly-SACKed */ 1230 1231 BUG_ON(!pcount); 1232 1233 /* Adjust counters and hints for the newly sacked sequence 1234 * range but discard the return value since prev is already 1235 * marked. We must tag the range first because the seq 1236 * advancement below implicitly advances 1237 * tcp_highest_sack_seq() when skb is highest_sack. 1238 */ 1239 tcp_sacktag_one(sk, state, TCP_SKB_CB(skb)->sacked, 1240 start_seq, end_seq, dup_sack, pcount, 1241 TCP_SKB_CB(skb)->when); 1242 1243 if (skb == tp->lost_skb_hint) 1244 tp->lost_cnt_hint += pcount; 1245 1246 TCP_SKB_CB(prev)->end_seq += shifted; 1247 TCP_SKB_CB(skb)->seq += shifted; 1248 1249 skb_shinfo(prev)->gso_segs += pcount; 1250 BUG_ON(skb_shinfo(skb)->gso_segs < pcount); 1251 skb_shinfo(skb)->gso_segs -= pcount; 1252 1253 /* When we're adding to gso_segs == 1, gso_size will be zero, 1254 * in theory this shouldn't be necessary but as long as DSACK 1255 * code can come after this skb later on it's better to keep 1256 * setting gso_size to something. 1257 */ 1258 if (!skb_shinfo(prev)->gso_size) { 1259 skb_shinfo(prev)->gso_size = mss; 1260 skb_shinfo(prev)->gso_type = sk->sk_gso_type; 1261 } 1262 1263 /* CHECKME: To clear or not to clear? Mimics normal skb currently */ 1264 if (skb_shinfo(skb)->gso_segs <= 1) { 1265 skb_shinfo(skb)->gso_size = 0; 1266 skb_shinfo(skb)->gso_type = 0; 1267 } 1268 1269 /* Difference in this won't matter, both ACKed by the same cumul. ACK */ 1270 TCP_SKB_CB(prev)->sacked |= (TCP_SKB_CB(skb)->sacked & TCPCB_EVER_RETRANS); 1271 1272 if (skb->len > 0) { 1273 BUG_ON(!tcp_skb_pcount(skb)); 1274 NET_INC_STATS_BH(sock_net(sk), LINUX_MIB_SACKSHIFTED); 1275 return false; 1276 } 1277 1278 /* Whole SKB was eaten :-) */ 1279 1280 if (skb == tp->retransmit_skb_hint) 1281 tp->retransmit_skb_hint = prev; 1282 if (skb == tp->lost_skb_hint) { 1283 tp->lost_skb_hint = prev; 1284 tp->lost_cnt_hint -= tcp_skb_pcount(prev); 1285 } 1286 1287 TCP_SKB_CB(skb)->tcp_flags |= TCP_SKB_CB(prev)->tcp_flags; 1288 if (skb == tcp_highest_sack(sk)) 1289 tcp_advance_highest_sack(sk, skb); 1290 1291 tcp_unlink_write_queue(skb, sk); 1292 sk_wmem_free_skb(sk, skb); 1293 1294 NET_INC_STATS_BH(sock_net(sk), LINUX_MIB_SACKMERGED); 1295 1296 return true; 1297 } 1298 1299 /* I wish gso_size would have a bit more sane initialization than 1300 * something-or-zero which complicates things 1301 */ 1302 static int tcp_skb_seglen(const struct sk_buff *skb) 1303 { 1304 return tcp_skb_pcount(skb) == 1 ? skb->len : tcp_skb_mss(skb); 1305 } 1306 1307 /* Shifting pages past head area doesn't work */ 1308 static int skb_can_shift(const struct sk_buff *skb) 1309 { 1310 return !skb_headlen(skb) && skb_is_nonlinear(skb); 1311 } 1312 1313 /* Try collapsing SACK blocks spanning across multiple skbs to a single 1314 * skb. 1315 */ 1316 static struct sk_buff *tcp_shift_skb_data(struct sock *sk, struct sk_buff *skb, 1317 struct tcp_sacktag_state *state, 1318 u32 start_seq, u32 end_seq, 1319 bool dup_sack) 1320 { 1321 struct tcp_sock *tp = tcp_sk(sk); 1322 struct sk_buff *prev; 1323 int mss; 1324 int pcount = 0; 1325 int len; 1326 int in_sack; 1327 1328 if (!sk_can_gso(sk)) 1329 goto fallback; 1330 1331 /* Normally R but no L won't result in plain S */ 1332 if (!dup_sack && 1333 (TCP_SKB_CB(skb)->sacked & (TCPCB_LOST|TCPCB_SACKED_RETRANS)) == TCPCB_SACKED_RETRANS) 1334 goto fallback; 1335 if (!skb_can_shift(skb)) 1336 goto fallback; 1337 /* This frame is about to be dropped (was ACKed). */ 1338 if (!after(TCP_SKB_CB(skb)->end_seq, tp->snd_una)) 1339 goto fallback; 1340 1341 /* Can only happen with delayed DSACK + discard craziness */ 1342 if (unlikely(skb == tcp_write_queue_head(sk))) 1343 goto fallback; 1344 prev = tcp_write_queue_prev(sk, skb); 1345 1346 if ((TCP_SKB_CB(prev)->sacked & TCPCB_TAGBITS) != TCPCB_SACKED_ACKED) 1347 goto fallback; 1348 1349 in_sack = !after(start_seq, TCP_SKB_CB(skb)->seq) && 1350 !before(end_seq, TCP_SKB_CB(skb)->end_seq); 1351 1352 if (in_sack) { 1353 len = skb->len; 1354 pcount = tcp_skb_pcount(skb); 1355 mss = tcp_skb_seglen(skb); 1356 1357 /* TODO: Fix DSACKs to not fragment already SACKed and we can 1358 * drop this restriction as unnecessary 1359 */ 1360 if (mss != tcp_skb_seglen(prev)) 1361 goto fallback; 1362 } else { 1363 if (!after(TCP_SKB_CB(skb)->end_seq, start_seq)) 1364 goto noop; 1365 /* CHECKME: This is non-MSS split case only?, this will 1366 * cause skipped skbs due to advancing loop btw, original 1367 * has that feature too 1368 */ 1369 if (tcp_skb_pcount(skb) <= 1) 1370 goto noop; 1371 1372 in_sack = !after(start_seq, TCP_SKB_CB(skb)->seq); 1373 if (!in_sack) { 1374 /* TODO: head merge to next could be attempted here 1375 * if (!after(TCP_SKB_CB(skb)->end_seq, end_seq)), 1376 * though it might not be worth of the additional hassle 1377 * 1378 * ...we can probably just fallback to what was done 1379 * previously. We could try merging non-SACKed ones 1380 * as well but it probably isn't going to buy off 1381 * because later SACKs might again split them, and 1382 * it would make skb timestamp tracking considerably 1383 * harder problem. 1384 */ 1385 goto fallback; 1386 } 1387 1388 len = end_seq - TCP_SKB_CB(skb)->seq; 1389 BUG_ON(len < 0); 1390 BUG_ON(len > skb->len); 1391 1392 /* MSS boundaries should be honoured or else pcount will 1393 * severely break even though it makes things bit trickier. 1394 * Optimize common case to avoid most of the divides 1395 */ 1396 mss = tcp_skb_mss(skb); 1397 1398 /* TODO: Fix DSACKs to not fragment already SACKed and we can 1399 * drop this restriction as unnecessary 1400 */ 1401 if (mss != tcp_skb_seglen(prev)) 1402 goto fallback; 1403 1404 if (len == mss) { 1405 pcount = 1; 1406 } else if (len < mss) { 1407 goto noop; 1408 } else { 1409 pcount = len / mss; 1410 len = pcount * mss; 1411 } 1412 } 1413 1414 /* tcp_sacktag_one() won't SACK-tag ranges below snd_una */ 1415 if (!after(TCP_SKB_CB(skb)->seq + len, tp->snd_una)) 1416 goto fallback; 1417 1418 if (!skb_shift(prev, skb, len)) 1419 goto fallback; 1420 if (!tcp_shifted_skb(sk, skb, state, pcount, len, mss, dup_sack)) 1421 goto out; 1422 1423 /* Hole filled allows collapsing with the next as well, this is very 1424 * useful when hole on every nth skb pattern happens 1425 */ 1426 if (prev == tcp_write_queue_tail(sk)) 1427 goto out; 1428 skb = tcp_write_queue_next(sk, prev); 1429 1430 if (!skb_can_shift(skb) || 1431 (skb == tcp_send_head(sk)) || 1432 ((TCP_SKB_CB(skb)->sacked & TCPCB_TAGBITS) != TCPCB_SACKED_ACKED) || 1433 (mss != tcp_skb_seglen(skb))) 1434 goto out; 1435 1436 len = skb->len; 1437 if (skb_shift(prev, skb, len)) { 1438 pcount += tcp_skb_pcount(skb); 1439 tcp_shifted_skb(sk, skb, state, tcp_skb_pcount(skb), len, mss, 0); 1440 } 1441 1442 out: 1443 state->fack_count += pcount; 1444 return prev; 1445 1446 noop: 1447 return skb; 1448 1449 fallback: 1450 NET_INC_STATS_BH(sock_net(sk), LINUX_MIB_SACKSHIFTFALLBACK); 1451 return NULL; 1452 } 1453 1454 static struct sk_buff *tcp_sacktag_walk(struct sk_buff *skb, struct sock *sk, 1455 struct tcp_sack_block *next_dup, 1456 struct tcp_sacktag_state *state, 1457 u32 start_seq, u32 end_seq, 1458 bool dup_sack_in) 1459 { 1460 struct tcp_sock *tp = tcp_sk(sk); 1461 struct sk_buff *tmp; 1462 1463 tcp_for_write_queue_from(skb, sk) { 1464 int in_sack = 0; 1465 bool dup_sack = dup_sack_in; 1466 1467 if (skb == tcp_send_head(sk)) 1468 break; 1469 1470 /* queue is in-order => we can short-circuit the walk early */ 1471 if (!before(TCP_SKB_CB(skb)->seq, end_seq)) 1472 break; 1473 1474 if ((next_dup != NULL) && 1475 before(TCP_SKB_CB(skb)->seq, next_dup->end_seq)) { 1476 in_sack = tcp_match_skb_to_sack(sk, skb, 1477 next_dup->start_seq, 1478 next_dup->end_seq); 1479 if (in_sack > 0) 1480 dup_sack = true; 1481 } 1482 1483 /* skb reference here is a bit tricky to get right, since 1484 * shifting can eat and free both this skb and the next, 1485 * so not even _safe variant of the loop is enough. 1486 */ 1487 if (in_sack <= 0) { 1488 tmp = tcp_shift_skb_data(sk, skb, state, 1489 start_seq, end_seq, dup_sack); 1490 if (tmp != NULL) { 1491 if (tmp != skb) { 1492 skb = tmp; 1493 continue; 1494 } 1495 1496 in_sack = 0; 1497 } else { 1498 in_sack = tcp_match_skb_to_sack(sk, skb, 1499 start_seq, 1500 end_seq); 1501 } 1502 } 1503 1504 if (unlikely(in_sack < 0)) 1505 break; 1506 1507 if (in_sack) { 1508 TCP_SKB_CB(skb)->sacked = 1509 tcp_sacktag_one(sk, 1510 state, 1511 TCP_SKB_CB(skb)->sacked, 1512 TCP_SKB_CB(skb)->seq, 1513 TCP_SKB_CB(skb)->end_seq, 1514 dup_sack, 1515 tcp_skb_pcount(skb), 1516 TCP_SKB_CB(skb)->when); 1517 1518 if (!before(TCP_SKB_CB(skb)->seq, 1519 tcp_highest_sack_seq(tp))) 1520 tcp_advance_highest_sack(sk, skb); 1521 } 1522 1523 state->fack_count += tcp_skb_pcount(skb); 1524 } 1525 return skb; 1526 } 1527 1528 /* Avoid all extra work that is being done by sacktag while walking in 1529 * a normal way 1530 */ 1531 static struct sk_buff *tcp_sacktag_skip(struct sk_buff *skb, struct sock *sk, 1532 struct tcp_sacktag_state *state, 1533 u32 skip_to_seq) 1534 { 1535 tcp_for_write_queue_from(skb, sk) { 1536 if (skb == tcp_send_head(sk)) 1537 break; 1538 1539 if (after(TCP_SKB_CB(skb)->end_seq, skip_to_seq)) 1540 break; 1541 1542 state->fack_count += tcp_skb_pcount(skb); 1543 } 1544 return skb; 1545 } 1546 1547 static struct sk_buff *tcp_maybe_skipping_dsack(struct sk_buff *skb, 1548 struct sock *sk, 1549 struct tcp_sack_block *next_dup, 1550 struct tcp_sacktag_state *state, 1551 u32 skip_to_seq) 1552 { 1553 if (next_dup == NULL) 1554 return skb; 1555 1556 if (before(next_dup->start_seq, skip_to_seq)) { 1557 skb = tcp_sacktag_skip(skb, sk, state, next_dup->start_seq); 1558 skb = tcp_sacktag_walk(skb, sk, NULL, state, 1559 next_dup->start_seq, next_dup->end_seq, 1560 1); 1561 } 1562 1563 return skb; 1564 } 1565 1566 static int tcp_sack_cache_ok(const struct tcp_sock *tp, const struct tcp_sack_block *cache) 1567 { 1568 return cache < tp->recv_sack_cache + ARRAY_SIZE(tp->recv_sack_cache); 1569 } 1570 1571 static int 1572 tcp_sacktag_write_queue(struct sock *sk, const struct sk_buff *ack_skb, 1573 u32 prior_snd_una, s32 *sack_rtt) 1574 { 1575 struct tcp_sock *tp = tcp_sk(sk); 1576 const unsigned char *ptr = (skb_transport_header(ack_skb) + 1577 TCP_SKB_CB(ack_skb)->sacked); 1578 struct tcp_sack_block_wire *sp_wire = (struct tcp_sack_block_wire *)(ptr+2); 1579 struct tcp_sack_block sp[TCP_NUM_SACKS]; 1580 struct tcp_sack_block *cache; 1581 struct tcp_sacktag_state state; 1582 struct sk_buff *skb; 1583 int num_sacks = min(TCP_NUM_SACKS, (ptr[1] - TCPOLEN_SACK_BASE) >> 3); 1584 int used_sacks; 1585 bool found_dup_sack = false; 1586 int i, j; 1587 int first_sack_index; 1588 1589 state.flag = 0; 1590 state.reord = tp->packets_out; 1591 state.rtt = -1; 1592 1593 if (!tp->sacked_out) { 1594 if (WARN_ON(tp->fackets_out)) 1595 tp->fackets_out = 0; 1596 tcp_highest_sack_reset(sk); 1597 } 1598 1599 found_dup_sack = tcp_check_dsack(sk, ack_skb, sp_wire, 1600 num_sacks, prior_snd_una); 1601 if (found_dup_sack) 1602 state.flag |= FLAG_DSACKING_ACK; 1603 1604 /* Eliminate too old ACKs, but take into 1605 * account more or less fresh ones, they can 1606 * contain valid SACK info. 1607 */ 1608 if (before(TCP_SKB_CB(ack_skb)->ack_seq, prior_snd_una - tp->max_window)) 1609 return 0; 1610 1611 if (!tp->packets_out) 1612 goto out; 1613 1614 used_sacks = 0; 1615 first_sack_index = 0; 1616 for (i = 0; i < num_sacks; i++) { 1617 bool dup_sack = !i && found_dup_sack; 1618 1619 sp[used_sacks].start_seq = get_unaligned_be32(&sp_wire[i].start_seq); 1620 sp[used_sacks].end_seq = get_unaligned_be32(&sp_wire[i].end_seq); 1621 1622 if (!tcp_is_sackblock_valid(tp, dup_sack, 1623 sp[used_sacks].start_seq, 1624 sp[used_sacks].end_seq)) { 1625 int mib_idx; 1626 1627 if (dup_sack) { 1628 if (!tp->undo_marker) 1629 mib_idx = LINUX_MIB_TCPDSACKIGNOREDNOUNDO; 1630 else 1631 mib_idx = LINUX_MIB_TCPDSACKIGNOREDOLD; 1632 } else { 1633 /* Don't count olds caused by ACK reordering */ 1634 if ((TCP_SKB_CB(ack_skb)->ack_seq != tp->snd_una) && 1635 !after(sp[used_sacks].end_seq, tp->snd_una)) 1636 continue; 1637 mib_idx = LINUX_MIB_TCPSACKDISCARD; 1638 } 1639 1640 NET_INC_STATS_BH(sock_net(sk), mib_idx); 1641 if (i == 0) 1642 first_sack_index = -1; 1643 continue; 1644 } 1645 1646 /* Ignore very old stuff early */ 1647 if (!after(sp[used_sacks].end_seq, prior_snd_una)) 1648 continue; 1649 1650 used_sacks++; 1651 } 1652 1653 /* order SACK blocks to allow in order walk of the retrans queue */ 1654 for (i = used_sacks - 1; i > 0; i--) { 1655 for (j = 0; j < i; j++) { 1656 if (after(sp[j].start_seq, sp[j + 1].start_seq)) { 1657 swap(sp[j], sp[j + 1]); 1658 1659 /* Track where the first SACK block goes to */ 1660 if (j == first_sack_index) 1661 first_sack_index = j + 1; 1662 } 1663 } 1664 } 1665 1666 skb = tcp_write_queue_head(sk); 1667 state.fack_count = 0; 1668 i = 0; 1669 1670 if (!tp->sacked_out) { 1671 /* It's already past, so skip checking against it */ 1672 cache = tp->recv_sack_cache + ARRAY_SIZE(tp->recv_sack_cache); 1673 } else { 1674 cache = tp->recv_sack_cache; 1675 /* Skip empty blocks in at head of the cache */ 1676 while (tcp_sack_cache_ok(tp, cache) && !cache->start_seq && 1677 !cache->end_seq) 1678 cache++; 1679 } 1680 1681 while (i < used_sacks) { 1682 u32 start_seq = sp[i].start_seq; 1683 u32 end_seq = sp[i].end_seq; 1684 bool dup_sack = (found_dup_sack && (i == first_sack_index)); 1685 struct tcp_sack_block *next_dup = NULL; 1686 1687 if (found_dup_sack && ((i + 1) == first_sack_index)) 1688 next_dup = &sp[i + 1]; 1689 1690 /* Skip too early cached blocks */ 1691 while (tcp_sack_cache_ok(tp, cache) && 1692 !before(start_seq, cache->end_seq)) 1693 cache++; 1694 1695 /* Can skip some work by looking recv_sack_cache? */ 1696 if (tcp_sack_cache_ok(tp, cache) && !dup_sack && 1697 after(end_seq, cache->start_seq)) { 1698 1699 /* Head todo? */ 1700 if (before(start_seq, cache->start_seq)) { 1701 skb = tcp_sacktag_skip(skb, sk, &state, 1702 start_seq); 1703 skb = tcp_sacktag_walk(skb, sk, next_dup, 1704 &state, 1705 start_seq, 1706 cache->start_seq, 1707 dup_sack); 1708 } 1709 1710 /* Rest of the block already fully processed? */ 1711 if (!after(end_seq, cache->end_seq)) 1712 goto advance_sp; 1713 1714 skb = tcp_maybe_skipping_dsack(skb, sk, next_dup, 1715 &state, 1716 cache->end_seq); 1717 1718 /* ...tail remains todo... */ 1719 if (tcp_highest_sack_seq(tp) == cache->end_seq) { 1720 /* ...but better entrypoint exists! */ 1721 skb = tcp_highest_sack(sk); 1722 if (skb == NULL) 1723 break; 1724 state.fack_count = tp->fackets_out; 1725 cache++; 1726 goto walk; 1727 } 1728 1729 skb = tcp_sacktag_skip(skb, sk, &state, cache->end_seq); 1730 /* Check overlap against next cached too (past this one already) */ 1731 cache++; 1732 continue; 1733 } 1734 1735 if (!before(start_seq, tcp_highest_sack_seq(tp))) { 1736 skb = tcp_highest_sack(sk); 1737 if (skb == NULL) 1738 break; 1739 state.fack_count = tp->fackets_out; 1740 } 1741 skb = tcp_sacktag_skip(skb, sk, &state, start_seq); 1742 1743 walk: 1744 skb = tcp_sacktag_walk(skb, sk, next_dup, &state, 1745 start_seq, end_seq, dup_sack); 1746 1747 advance_sp: 1748 i++; 1749 } 1750 1751 /* Clear the head of the cache sack blocks so we can skip it next time */ 1752 for (i = 0; i < ARRAY_SIZE(tp->recv_sack_cache) - used_sacks; i++) { 1753 tp->recv_sack_cache[i].start_seq = 0; 1754 tp->recv_sack_cache[i].end_seq = 0; 1755 } 1756 for (j = 0; j < used_sacks; j++) 1757 tp->recv_sack_cache[i++] = sp[j]; 1758 1759 tcp_mark_lost_retrans(sk); 1760 1761 tcp_verify_left_out(tp); 1762 1763 if ((state.reord < tp->fackets_out) && 1764 ((inet_csk(sk)->icsk_ca_state != TCP_CA_Loss) || tp->undo_marker)) 1765 tcp_update_reordering(sk, tp->fackets_out - state.reord, 0); 1766 1767 out: 1768 1769 #if FASTRETRANS_DEBUG > 0 1770 WARN_ON((int)tp->sacked_out < 0); 1771 WARN_ON((int)tp->lost_out < 0); 1772 WARN_ON((int)tp->retrans_out < 0); 1773 WARN_ON((int)tcp_packets_in_flight(tp) < 0); 1774 #endif 1775 *sack_rtt = state.rtt; 1776 return state.flag; 1777 } 1778 1779 /* Limits sacked_out so that sum with lost_out isn't ever larger than 1780 * packets_out. Returns false if sacked_out adjustement wasn't necessary. 1781 */ 1782 static bool tcp_limit_reno_sacked(struct tcp_sock *tp) 1783 { 1784 u32 holes; 1785 1786 holes = max(tp->lost_out, 1U); 1787 holes = min(holes, tp->packets_out); 1788 1789 if ((tp->sacked_out + holes) > tp->packets_out) { 1790 tp->sacked_out = tp->packets_out - holes; 1791 return true; 1792 } 1793 return false; 1794 } 1795 1796 /* If we receive more dupacks than we expected counting segments 1797 * in assumption of absent reordering, interpret this as reordering. 1798 * The only another reason could be bug in receiver TCP. 1799 */ 1800 static void tcp_check_reno_reordering(struct sock *sk, const int addend) 1801 { 1802 struct tcp_sock *tp = tcp_sk(sk); 1803 if (tcp_limit_reno_sacked(tp)) 1804 tcp_update_reordering(sk, tp->packets_out + addend, 0); 1805 } 1806 1807 /* Emulate SACKs for SACKless connection: account for a new dupack. */ 1808 1809 static void tcp_add_reno_sack(struct sock *sk) 1810 { 1811 struct tcp_sock *tp = tcp_sk(sk); 1812 tp->sacked_out++; 1813 tcp_check_reno_reordering(sk, 0); 1814 tcp_verify_left_out(tp); 1815 } 1816 1817 /* Account for ACK, ACKing some data in Reno Recovery phase. */ 1818 1819 static void tcp_remove_reno_sacks(struct sock *sk, int acked) 1820 { 1821 struct tcp_sock *tp = tcp_sk(sk); 1822 1823 if (acked > 0) { 1824 /* One ACK acked hole. The rest eat duplicate ACKs. */ 1825 if (acked - 1 >= tp->sacked_out) 1826 tp->sacked_out = 0; 1827 else 1828 tp->sacked_out -= acked - 1; 1829 } 1830 tcp_check_reno_reordering(sk, acked); 1831 tcp_verify_left_out(tp); 1832 } 1833 1834 static inline void tcp_reset_reno_sack(struct tcp_sock *tp) 1835 { 1836 tp->sacked_out = 0; 1837 } 1838 1839 static void tcp_clear_retrans_partial(struct tcp_sock *tp) 1840 { 1841 tp->retrans_out = 0; 1842 tp->lost_out = 0; 1843 1844 tp->undo_marker = 0; 1845 tp->undo_retrans = 0; 1846 } 1847 1848 void tcp_clear_retrans(struct tcp_sock *tp) 1849 { 1850 tcp_clear_retrans_partial(tp); 1851 1852 tp->fackets_out = 0; 1853 tp->sacked_out = 0; 1854 } 1855 1856 /* Enter Loss state. If "how" is not zero, forget all SACK information 1857 * and reset tags completely, otherwise preserve SACKs. If receiver 1858 * dropped its ofo queue, we will know this due to reneging detection. 1859 */ 1860 void tcp_enter_loss(struct sock *sk, int how) 1861 { 1862 const struct inet_connection_sock *icsk = inet_csk(sk); 1863 struct tcp_sock *tp = tcp_sk(sk); 1864 struct sk_buff *skb; 1865 bool new_recovery = false; 1866 1867 /* Reduce ssthresh if it has not yet been made inside this window. */ 1868 if (icsk->icsk_ca_state <= TCP_CA_Disorder || 1869 !after(tp->high_seq, tp->snd_una) || 1870 (icsk->icsk_ca_state == TCP_CA_Loss && !icsk->icsk_retransmits)) { 1871 new_recovery = true; 1872 tp->prior_ssthresh = tcp_current_ssthresh(sk); 1873 tp->snd_ssthresh = icsk->icsk_ca_ops->ssthresh(sk); 1874 tcp_ca_event(sk, CA_EVENT_LOSS); 1875 } 1876 tp->snd_cwnd = 1; 1877 tp->snd_cwnd_cnt = 0; 1878 tp->snd_cwnd_stamp = tcp_time_stamp; 1879 1880 tcp_clear_retrans_partial(tp); 1881 1882 if (tcp_is_reno(tp)) 1883 tcp_reset_reno_sack(tp); 1884 1885 tp->undo_marker = tp->snd_una; 1886 if (how) { 1887 tp->sacked_out = 0; 1888 tp->fackets_out = 0; 1889 } 1890 tcp_clear_all_retrans_hints(tp); 1891 1892 tcp_for_write_queue(skb, sk) { 1893 if (skb == tcp_send_head(sk)) 1894 break; 1895 1896 if (TCP_SKB_CB(skb)->sacked & TCPCB_RETRANS) 1897 tp->undo_marker = 0; 1898 TCP_SKB_CB(skb)->sacked &= (~TCPCB_TAGBITS)|TCPCB_SACKED_ACKED; 1899 if (!(TCP_SKB_CB(skb)->sacked&TCPCB_SACKED_ACKED) || how) { 1900 TCP_SKB_CB(skb)->sacked &= ~TCPCB_SACKED_ACKED; 1901 TCP_SKB_CB(skb)->sacked |= TCPCB_LOST; 1902 tp->lost_out += tcp_skb_pcount(skb); 1903 tp->retransmit_high = TCP_SKB_CB(skb)->end_seq; 1904 } 1905 } 1906 tcp_verify_left_out(tp); 1907 1908 /* Timeout in disordered state after receiving substantial DUPACKs 1909 * suggests that the degree of reordering is over-estimated. 1910 */ 1911 if (icsk->icsk_ca_state <= TCP_CA_Disorder && 1912 tp->sacked_out >= sysctl_tcp_reordering) 1913 tp->reordering = min_t(unsigned int, tp->reordering, 1914 sysctl_tcp_reordering); 1915 tcp_set_ca_state(sk, TCP_CA_Loss); 1916 tp->high_seq = tp->snd_nxt; 1917 TCP_ECN_queue_cwr(tp); 1918 1919 /* F-RTO RFC5682 sec 3.1 step 1: retransmit SND.UNA if no previous 1920 * loss recovery is underway except recurring timeout(s) on 1921 * the same SND.UNA (sec 3.2). Disable F-RTO on path MTU probing 1922 */ 1923 tp->frto = sysctl_tcp_frto && 1924 (new_recovery || icsk->icsk_retransmits) && 1925 !inet_csk(sk)->icsk_mtup.probe_size; 1926 } 1927 1928 /* If ACK arrived pointing to a remembered SACK, it means that our 1929 * remembered SACKs do not reflect real state of receiver i.e. 1930 * receiver _host_ is heavily congested (or buggy). 1931 * 1932 * Do processing similar to RTO timeout. 1933 */ 1934 static bool tcp_check_sack_reneging(struct sock *sk, int flag) 1935 { 1936 if (flag & FLAG_SACK_RENEGING) { 1937 struct inet_connection_sock *icsk = inet_csk(sk); 1938 NET_INC_STATS_BH(sock_net(sk), LINUX_MIB_TCPSACKRENEGING); 1939 1940 tcp_enter_loss(sk, 1); 1941 icsk->icsk_retransmits++; 1942 tcp_retransmit_skb(sk, tcp_write_queue_head(sk)); 1943 inet_csk_reset_xmit_timer(sk, ICSK_TIME_RETRANS, 1944 icsk->icsk_rto, TCP_RTO_MAX); 1945 return true; 1946 } 1947 return false; 1948 } 1949 1950 static inline int tcp_fackets_out(const struct tcp_sock *tp) 1951 { 1952 return tcp_is_reno(tp) ? tp->sacked_out + 1 : tp->fackets_out; 1953 } 1954 1955 /* Heurestics to calculate number of duplicate ACKs. There's no dupACKs 1956 * counter when SACK is enabled (without SACK, sacked_out is used for 1957 * that purpose). 1958 * 1959 * Instead, with FACK TCP uses fackets_out that includes both SACKed 1960 * segments up to the highest received SACK block so far and holes in 1961 * between them. 1962 * 1963 * With reordering, holes may still be in flight, so RFC3517 recovery 1964 * uses pure sacked_out (total number of SACKed segments) even though 1965 * it violates the RFC that uses duplicate ACKs, often these are equal 1966 * but when e.g. out-of-window ACKs or packet duplication occurs, 1967 * they differ. Since neither occurs due to loss, TCP should really 1968 * ignore them. 1969 */ 1970 static inline int tcp_dupack_heuristics(const struct tcp_sock *tp) 1971 { 1972 return tcp_is_fack(tp) ? tp->fackets_out : tp->sacked_out + 1; 1973 } 1974 1975 static bool tcp_pause_early_retransmit(struct sock *sk, int flag) 1976 { 1977 struct tcp_sock *tp = tcp_sk(sk); 1978 unsigned long delay; 1979 1980 /* Delay early retransmit and entering fast recovery for 1981 * max(RTT/4, 2msec) unless ack has ECE mark, no RTT samples 1982 * available, or RTO is scheduled to fire first. 1983 */ 1984 if (sysctl_tcp_early_retrans < 2 || sysctl_tcp_early_retrans > 3 || 1985 (flag & FLAG_ECE) || !tp->srtt) 1986 return false; 1987 1988 delay = max_t(unsigned long, (tp->srtt >> 5), msecs_to_jiffies(2)); 1989 if (!time_after(inet_csk(sk)->icsk_timeout, (jiffies + delay))) 1990 return false; 1991 1992 inet_csk_reset_xmit_timer(sk, ICSK_TIME_EARLY_RETRANS, delay, 1993 TCP_RTO_MAX); 1994 return true; 1995 } 1996 1997 /* Linux NewReno/SACK/FACK/ECN state machine. 1998 * -------------------------------------- 1999 * 2000 * "Open" Normal state, no dubious events, fast path. 2001 * "Disorder" In all the respects it is "Open", 2002 * but requires a bit more attention. It is entered when 2003 * we see some SACKs or dupacks. It is split of "Open" 2004 * mainly to move some processing from fast path to slow one. 2005 * "CWR" CWND was reduced due to some Congestion Notification event. 2006 * It can be ECN, ICMP source quench, local device congestion. 2007 * "Recovery" CWND was reduced, we are fast-retransmitting. 2008 * "Loss" CWND was reduced due to RTO timeout or SACK reneging. 2009 * 2010 * tcp_fastretrans_alert() is entered: 2011 * - each incoming ACK, if state is not "Open" 2012 * - when arrived ACK is unusual, namely: 2013 * * SACK 2014 * * Duplicate ACK. 2015 * * ECN ECE. 2016 * 2017 * Counting packets in flight is pretty simple. 2018 * 2019 * in_flight = packets_out - left_out + retrans_out 2020 * 2021 * packets_out is SND.NXT-SND.UNA counted in packets. 2022 * 2023 * retrans_out is number of retransmitted segments. 2024 * 2025 * left_out is number of segments left network, but not ACKed yet. 2026 * 2027 * left_out = sacked_out + lost_out 2028 * 2029 * sacked_out: Packets, which arrived to receiver out of order 2030 * and hence not ACKed. With SACKs this number is simply 2031 * amount of SACKed data. Even without SACKs 2032 * it is easy to give pretty reliable estimate of this number, 2033 * counting duplicate ACKs. 2034 * 2035 * lost_out: Packets lost by network. TCP has no explicit 2036 * "loss notification" feedback from network (for now). 2037 * It means that this number can be only _guessed_. 2038 * Actually, it is the heuristics to predict lossage that 2039 * distinguishes different algorithms. 2040 * 2041 * F.e. after RTO, when all the queue is considered as lost, 2042 * lost_out = packets_out and in_flight = retrans_out. 2043 * 2044 * Essentially, we have now two algorithms counting 2045 * lost packets. 2046 * 2047 * FACK: It is the simplest heuristics. As soon as we decided 2048 * that something is lost, we decide that _all_ not SACKed 2049 * packets until the most forward SACK are lost. I.e. 2050 * lost_out = fackets_out - sacked_out and left_out = fackets_out. 2051 * It is absolutely correct estimate, if network does not reorder 2052 * packets. And it loses any connection to reality when reordering 2053 * takes place. We use FACK by default until reordering 2054 * is suspected on the path to this destination. 2055 * 2056 * NewReno: when Recovery is entered, we assume that one segment 2057 * is lost (classic Reno). While we are in Recovery and 2058 * a partial ACK arrives, we assume that one more packet 2059 * is lost (NewReno). This heuristics are the same in NewReno 2060 * and SACK. 2061 * 2062 * Imagine, that's all! Forget about all this shamanism about CWND inflation 2063 * deflation etc. CWND is real congestion window, never inflated, changes 2064 * only according to classic VJ rules. 2065 * 2066 * Really tricky (and requiring careful tuning) part of algorithm 2067 * is hidden in functions tcp_time_to_recover() and tcp_xmit_retransmit_queue(). 2068 * The first determines the moment _when_ we should reduce CWND and, 2069 * hence, slow down forward transmission. In fact, it determines the moment 2070 * when we decide that hole is caused by loss, rather than by a reorder. 2071 * 2072 * tcp_xmit_retransmit_queue() decides, _what_ we should retransmit to fill 2073 * holes, caused by lost packets. 2074 * 2075 * And the most logically complicated part of algorithm is undo 2076 * heuristics. We detect false retransmits due to both too early 2077 * fast retransmit (reordering) and underestimated RTO, analyzing 2078 * timestamps and D-SACKs. When we detect that some segments were 2079 * retransmitted by mistake and CWND reduction was wrong, we undo 2080 * window reduction and abort recovery phase. This logic is hidden 2081 * inside several functions named tcp_try_undo_<something>. 2082 */ 2083 2084 /* This function decides, when we should leave Disordered state 2085 * and enter Recovery phase, reducing congestion window. 2086 * 2087 * Main question: may we further continue forward transmission 2088 * with the same cwnd? 2089 */ 2090 static bool tcp_time_to_recover(struct sock *sk, int flag) 2091 { 2092 struct tcp_sock *tp = tcp_sk(sk); 2093 __u32 packets_out; 2094 2095 /* Trick#1: The loss is proven. */ 2096 if (tp->lost_out) 2097 return true; 2098 2099 /* Not-A-Trick#2 : Classic rule... */ 2100 if (tcp_dupack_heuristics(tp) > tp->reordering) 2101 return true; 2102 2103 /* Trick#4: It is still not OK... But will it be useful to delay 2104 * recovery more? 2105 */ 2106 packets_out = tp->packets_out; 2107 if (packets_out <= tp->reordering && 2108 tp->sacked_out >= max_t(__u32, packets_out/2, sysctl_tcp_reordering) && 2109 !tcp_may_send_now(sk)) { 2110 /* We have nothing to send. This connection is limited 2111 * either by receiver window or by application. 2112 */ 2113 return true; 2114 } 2115 2116 /* If a thin stream is detected, retransmit after first 2117 * received dupack. Employ only if SACK is supported in order 2118 * to avoid possible corner-case series of spurious retransmissions 2119 * Use only if there are no unsent data. 2120 */ 2121 if ((tp->thin_dupack || sysctl_tcp_thin_dupack) && 2122 tcp_stream_is_thin(tp) && tcp_dupack_heuristics(tp) > 1 && 2123 tcp_is_sack(tp) && !tcp_send_head(sk)) 2124 return true; 2125 2126 /* Trick#6: TCP early retransmit, per RFC5827. To avoid spurious 2127 * retransmissions due to small network reorderings, we implement 2128 * Mitigation A.3 in the RFC and delay the retransmission for a short 2129 * interval if appropriate. 2130 */ 2131 if (tp->do_early_retrans && !tp->retrans_out && tp->sacked_out && 2132 (tp->packets_out >= (tp->sacked_out + 1) && tp->packets_out < 4) && 2133 !tcp_may_send_now(sk)) 2134 return !tcp_pause_early_retransmit(sk, flag); 2135 2136 return false; 2137 } 2138 2139 /* Detect loss in event "A" above by marking head of queue up as lost. 2140 * For FACK or non-SACK(Reno) senders, the first "packets" number of segments 2141 * are considered lost. For RFC3517 SACK, a segment is considered lost if it 2142 * has at least tp->reordering SACKed seqments above it; "packets" refers to 2143 * the maximum SACKed segments to pass before reaching this limit. 2144 */ 2145 static void tcp_mark_head_lost(struct sock *sk, int packets, int mark_head) 2146 { 2147 struct tcp_sock *tp = tcp_sk(sk); 2148 struct sk_buff *skb; 2149 int cnt, oldcnt; 2150 int err; 2151 unsigned int mss; 2152 /* Use SACK to deduce losses of new sequences sent during recovery */ 2153 const u32 loss_high = tcp_is_sack(tp) ? tp->snd_nxt : tp->high_seq; 2154 2155 WARN_ON(packets > tp->packets_out); 2156 if (tp->lost_skb_hint) { 2157 skb = tp->lost_skb_hint; 2158 cnt = tp->lost_cnt_hint; 2159 /* Head already handled? */ 2160 if (mark_head && skb != tcp_write_queue_head(sk)) 2161 return; 2162 } else { 2163 skb = tcp_write_queue_head(sk); 2164 cnt = 0; 2165 } 2166 2167 tcp_for_write_queue_from(skb, sk) { 2168 if (skb == tcp_send_head(sk)) 2169 break; 2170 /* TODO: do this better */ 2171 /* this is not the most efficient way to do this... */ 2172 tp->lost_skb_hint = skb; 2173 tp->lost_cnt_hint = cnt; 2174 2175 if (after(TCP_SKB_CB(skb)->end_seq, loss_high)) 2176 break; 2177 2178 oldcnt = cnt; 2179 if (tcp_is_fack(tp) || tcp_is_reno(tp) || 2180 (TCP_SKB_CB(skb)->sacked & TCPCB_SACKED_ACKED)) 2181 cnt += tcp_skb_pcount(skb); 2182 2183 if (cnt > packets) { 2184 if ((tcp_is_sack(tp) && !tcp_is_fack(tp)) || 2185 (TCP_SKB_CB(skb)->sacked & TCPCB_SACKED_ACKED) || 2186 (oldcnt >= packets)) 2187 break; 2188 2189 mss = skb_shinfo(skb)->gso_size; 2190 err = tcp_fragment(sk, skb, (packets - oldcnt) * mss, mss); 2191 if (err < 0) 2192 break; 2193 cnt = packets; 2194 } 2195 2196 tcp_skb_mark_lost(tp, skb); 2197 2198 if (mark_head) 2199 break; 2200 } 2201 tcp_verify_left_out(tp); 2202 } 2203 2204 /* Account newly detected lost packet(s) */ 2205 2206 static void tcp_update_scoreboard(struct sock *sk, int fast_rexmit) 2207 { 2208 struct tcp_sock *tp = tcp_sk(sk); 2209 2210 if (tcp_is_reno(tp)) { 2211 tcp_mark_head_lost(sk, 1, 1); 2212 } else if (tcp_is_fack(tp)) { 2213 int lost = tp->fackets_out - tp->reordering; 2214 if (lost <= 0) 2215 lost = 1; 2216 tcp_mark_head_lost(sk, lost, 0); 2217 } else { 2218 int sacked_upto = tp->sacked_out - tp->reordering; 2219 if (sacked_upto >= 0) 2220 tcp_mark_head_lost(sk, sacked_upto, 0); 2221 else if (fast_rexmit) 2222 tcp_mark_head_lost(sk, 1, 1); 2223 } 2224 } 2225 2226 /* CWND moderation, preventing bursts due to too big ACKs 2227 * in dubious situations. 2228 */ 2229 static inline void tcp_moderate_cwnd(struct tcp_sock *tp) 2230 { 2231 tp->snd_cwnd = min(tp->snd_cwnd, 2232 tcp_packets_in_flight(tp) + tcp_max_burst(tp)); 2233 tp->snd_cwnd_stamp = tcp_time_stamp; 2234 } 2235 2236 /* Nothing was retransmitted or returned timestamp is less 2237 * than timestamp of the first retransmission. 2238 */ 2239 static inline bool tcp_packet_delayed(const struct tcp_sock *tp) 2240 { 2241 return !tp->retrans_stamp || 2242 (tp->rx_opt.saw_tstamp && tp->rx_opt.rcv_tsecr && 2243 before(tp->rx_opt.rcv_tsecr, tp->retrans_stamp)); 2244 } 2245 2246 /* Undo procedures. */ 2247 2248 #if FASTRETRANS_DEBUG > 1 2249 static void DBGUNDO(struct sock *sk, const char *msg) 2250 { 2251 struct tcp_sock *tp = tcp_sk(sk); 2252 struct inet_sock *inet = inet_sk(sk); 2253 2254 if (sk->sk_family == AF_INET) { 2255 pr_debug("Undo %s %pI4/%u c%u l%u ss%u/%u p%u\n", 2256 msg, 2257 &inet->inet_daddr, ntohs(inet->inet_dport), 2258 tp->snd_cwnd, tcp_left_out(tp), 2259 tp->snd_ssthresh, tp->prior_ssthresh, 2260 tp->packets_out); 2261 } 2262 #if IS_ENABLED(CONFIG_IPV6) 2263 else if (sk->sk_family == AF_INET6) { 2264 struct ipv6_pinfo *np = inet6_sk(sk); 2265 pr_debug("Undo %s %pI6/%u c%u l%u ss%u/%u p%u\n", 2266 msg, 2267 &np->daddr, ntohs(inet->inet_dport), 2268 tp->snd_cwnd, tcp_left_out(tp), 2269 tp->snd_ssthresh, tp->prior_ssthresh, 2270 tp->packets_out); 2271 } 2272 #endif 2273 } 2274 #else 2275 #define DBGUNDO(x...) do { } while (0) 2276 #endif 2277 2278 static void tcp_undo_cwnd_reduction(struct sock *sk, bool unmark_loss) 2279 { 2280 struct tcp_sock *tp = tcp_sk(sk); 2281 2282 if (unmark_loss) { 2283 struct sk_buff *skb; 2284 2285 tcp_for_write_queue(skb, sk) { 2286 if (skb == tcp_send_head(sk)) 2287 break; 2288 TCP_SKB_CB(skb)->sacked &= ~TCPCB_LOST; 2289 } 2290 tp->lost_out = 0; 2291 tcp_clear_all_retrans_hints(tp); 2292 } 2293 2294 if (tp->prior_ssthresh) { 2295 const struct inet_connection_sock *icsk = inet_csk(sk); 2296 2297 if (icsk->icsk_ca_ops->undo_cwnd) 2298 tp->snd_cwnd = icsk->icsk_ca_ops->undo_cwnd(sk); 2299 else 2300 tp->snd_cwnd = max(tp->snd_cwnd, tp->snd_ssthresh << 1); 2301 2302 if (tp->prior_ssthresh > tp->snd_ssthresh) { 2303 tp->snd_ssthresh = tp->prior_ssthresh; 2304 TCP_ECN_withdraw_cwr(tp); 2305 } 2306 } else { 2307 tp->snd_cwnd = max(tp->snd_cwnd, tp->snd_ssthresh); 2308 } 2309 tp->snd_cwnd_stamp = tcp_time_stamp; 2310 tp->undo_marker = 0; 2311 } 2312 2313 static inline bool tcp_may_undo(const struct tcp_sock *tp) 2314 { 2315 return tp->undo_marker && (!tp->undo_retrans || tcp_packet_delayed(tp)); 2316 } 2317 2318 /* People celebrate: "We love our President!" */ 2319 static bool tcp_try_undo_recovery(struct sock *sk) 2320 { 2321 struct tcp_sock *tp = tcp_sk(sk); 2322 2323 if (tcp_may_undo(tp)) { 2324 int mib_idx; 2325 2326 /* Happy end! We did not retransmit anything 2327 * or our original transmission succeeded. 2328 */ 2329 DBGUNDO(sk, inet_csk(sk)->icsk_ca_state == TCP_CA_Loss ? "loss" : "retrans"); 2330 tcp_undo_cwnd_reduction(sk, false); 2331 if (inet_csk(sk)->icsk_ca_state == TCP_CA_Loss) 2332 mib_idx = LINUX_MIB_TCPLOSSUNDO; 2333 else 2334 mib_idx = LINUX_MIB_TCPFULLUNDO; 2335 2336 NET_INC_STATS_BH(sock_net(sk), mib_idx); 2337 } 2338 if (tp->snd_una == tp->high_seq && tcp_is_reno(tp)) { 2339 /* Hold old state until something *above* high_seq 2340 * is ACKed. For Reno it is MUST to prevent false 2341 * fast retransmits (RFC2582). SACK TCP is safe. */ 2342 tcp_moderate_cwnd(tp); 2343 return true; 2344 } 2345 tcp_set_ca_state(sk, TCP_CA_Open); 2346 return false; 2347 } 2348 2349 /* Try to undo cwnd reduction, because D-SACKs acked all retransmitted data */ 2350 static bool tcp_try_undo_dsack(struct sock *sk) 2351 { 2352 struct tcp_sock *tp = tcp_sk(sk); 2353 2354 if (tp->undo_marker && !tp->undo_retrans) { 2355 DBGUNDO(sk, "D-SACK"); 2356 tcp_undo_cwnd_reduction(sk, false); 2357 NET_INC_STATS_BH(sock_net(sk), LINUX_MIB_TCPDSACKUNDO); 2358 return true; 2359 } 2360 return false; 2361 } 2362 2363 /* We can clear retrans_stamp when there are no retransmissions in the 2364 * window. It would seem that it is trivially available for us in 2365 * tp->retrans_out, however, that kind of assumptions doesn't consider 2366 * what will happen if errors occur when sending retransmission for the 2367 * second time. ...It could the that such segment has only 2368 * TCPCB_EVER_RETRANS set at the present time. It seems that checking 2369 * the head skb is enough except for some reneging corner cases that 2370 * are not worth the effort. 2371 * 2372 * Main reason for all this complexity is the fact that connection dying 2373 * time now depends on the validity of the retrans_stamp, in particular, 2374 * that successive retransmissions of a segment must not advance 2375 * retrans_stamp under any conditions. 2376 */ 2377 static bool tcp_any_retrans_done(const struct sock *sk) 2378 { 2379 const struct tcp_sock *tp = tcp_sk(sk); 2380 struct sk_buff *skb; 2381 2382 if (tp->retrans_out) 2383 return true; 2384 2385 skb = tcp_write_queue_head(sk); 2386 if (unlikely(skb && TCP_SKB_CB(skb)->sacked & TCPCB_EVER_RETRANS)) 2387 return true; 2388 2389 return false; 2390 } 2391 2392 /* Undo during loss recovery after partial ACK or using F-RTO. */ 2393 static bool tcp_try_undo_loss(struct sock *sk, bool frto_undo) 2394 { 2395 struct tcp_sock *tp = tcp_sk(sk); 2396 2397 if (frto_undo || tcp_may_undo(tp)) { 2398 tcp_undo_cwnd_reduction(sk, true); 2399 2400 DBGUNDO(sk, "partial loss"); 2401 NET_INC_STATS_BH(sock_net(sk), LINUX_MIB_TCPLOSSUNDO); 2402 if (frto_undo) 2403 NET_INC_STATS_BH(sock_net(sk), 2404 LINUX_MIB_TCPSPURIOUSRTOS); 2405 inet_csk(sk)->icsk_retransmits = 0; 2406 if (frto_undo || tcp_is_sack(tp)) 2407 tcp_set_ca_state(sk, TCP_CA_Open); 2408 return true; 2409 } 2410 return false; 2411 } 2412 2413 /* The cwnd reduction in CWR and Recovery use the PRR algorithm 2414 * https://datatracker.ietf.org/doc/draft-ietf-tcpm-proportional-rate-reduction/ 2415 * It computes the number of packets to send (sndcnt) based on packets newly 2416 * delivered: 2417 * 1) If the packets in flight is larger than ssthresh, PRR spreads the 2418 * cwnd reductions across a full RTT. 2419 * 2) If packets in flight is lower than ssthresh (such as due to excess 2420 * losses and/or application stalls), do not perform any further cwnd 2421 * reductions, but instead slow start up to ssthresh. 2422 */ 2423 static void tcp_init_cwnd_reduction(struct sock *sk, const bool set_ssthresh) 2424 { 2425 struct tcp_sock *tp = tcp_sk(sk); 2426 2427 tp->high_seq = tp->snd_nxt; 2428 tp->tlp_high_seq = 0; 2429 tp->snd_cwnd_cnt = 0; 2430 tp->prior_cwnd = tp->snd_cwnd; 2431 tp->prr_delivered = 0; 2432 tp->prr_out = 0; 2433 if (set_ssthresh) 2434 tp->snd_ssthresh = inet_csk(sk)->icsk_ca_ops->ssthresh(sk); 2435 TCP_ECN_queue_cwr(tp); 2436 } 2437 2438 static void tcp_cwnd_reduction(struct sock *sk, const int prior_unsacked, 2439 int fast_rexmit) 2440 { 2441 struct tcp_sock *tp = tcp_sk(sk); 2442 int sndcnt = 0; 2443 int delta = tp->snd_ssthresh - tcp_packets_in_flight(tp); 2444 int newly_acked_sacked = prior_unsacked - 2445 (tp->packets_out - tp->sacked_out); 2446 2447 tp->prr_delivered += newly_acked_sacked; 2448 if (tcp_packets_in_flight(tp) > tp->snd_ssthresh) { 2449 u64 dividend = (u64)tp->snd_ssthresh * tp->prr_delivered + 2450 tp->prior_cwnd - 1; 2451 sndcnt = div_u64(dividend, tp->prior_cwnd) - tp->prr_out; 2452 } else { 2453 sndcnt = min_t(int, delta, 2454 max_t(int, tp->prr_delivered - tp->prr_out, 2455 newly_acked_sacked) + 1); 2456 } 2457 2458 sndcnt = max(sndcnt, (fast_rexmit ? 1 : 0)); 2459 tp->snd_cwnd = tcp_packets_in_flight(tp) + sndcnt; 2460 } 2461 2462 static inline void tcp_end_cwnd_reduction(struct sock *sk) 2463 { 2464 struct tcp_sock *tp = tcp_sk(sk); 2465 2466 /* Reset cwnd to ssthresh in CWR or Recovery (unless it's undone) */ 2467 if (inet_csk(sk)->icsk_ca_state == TCP_CA_CWR || 2468 (tp->undo_marker && tp->snd_ssthresh < TCP_INFINITE_SSTHRESH)) { 2469 tp->snd_cwnd = tp->snd_ssthresh; 2470 tp->snd_cwnd_stamp = tcp_time_stamp; 2471 } 2472 tcp_ca_event(sk, CA_EVENT_COMPLETE_CWR); 2473 } 2474 2475 /* Enter CWR state. Disable cwnd undo since congestion is proven with ECN */ 2476 void tcp_enter_cwr(struct sock *sk, const int set_ssthresh) 2477 { 2478 struct tcp_sock *tp = tcp_sk(sk); 2479 2480 tp->prior_ssthresh = 0; 2481 if (inet_csk(sk)->icsk_ca_state < TCP_CA_CWR) { 2482 tp->undo_marker = 0; 2483 tcp_init_cwnd_reduction(sk, set_ssthresh); 2484 tcp_set_ca_state(sk, TCP_CA_CWR); 2485 } 2486 } 2487 2488 static void tcp_try_keep_open(struct sock *sk) 2489 { 2490 struct tcp_sock *tp = tcp_sk(sk); 2491 int state = TCP_CA_Open; 2492 2493 if (tcp_left_out(tp) || tcp_any_retrans_done(sk)) 2494 state = TCP_CA_Disorder; 2495 2496 if (inet_csk(sk)->icsk_ca_state != state) { 2497 tcp_set_ca_state(sk, state); 2498 tp->high_seq = tp->snd_nxt; 2499 } 2500 } 2501 2502 static void tcp_try_to_open(struct sock *sk, int flag, const int prior_unsacked) 2503 { 2504 struct tcp_sock *tp = tcp_sk(sk); 2505 2506 tcp_verify_left_out(tp); 2507 2508 if (!tcp_any_retrans_done(sk)) 2509 tp->retrans_stamp = 0; 2510 2511 if (flag & FLAG_ECE) 2512 tcp_enter_cwr(sk, 1); 2513 2514 if (inet_csk(sk)->icsk_ca_state != TCP_CA_CWR) { 2515 tcp_try_keep_open(sk); 2516 } else { 2517 tcp_cwnd_reduction(sk, prior_unsacked, 0); 2518 } 2519 } 2520 2521 static void tcp_mtup_probe_failed(struct sock *sk) 2522 { 2523 struct inet_connection_sock *icsk = inet_csk(sk); 2524 2525 icsk->icsk_mtup.search_high = icsk->icsk_mtup.probe_size - 1; 2526 icsk->icsk_mtup.probe_size = 0; 2527 } 2528 2529 static void tcp_mtup_probe_success(struct sock *sk) 2530 { 2531 struct tcp_sock *tp = tcp_sk(sk); 2532 struct inet_connection_sock *icsk = inet_csk(sk); 2533 2534 /* FIXME: breaks with very large cwnd */ 2535 tp->prior_ssthresh = tcp_current_ssthresh(sk); 2536 tp->snd_cwnd = tp->snd_cwnd * 2537 tcp_mss_to_mtu(sk, tp->mss_cache) / 2538 icsk->icsk_mtup.probe_size; 2539 tp->snd_cwnd_cnt = 0; 2540 tp->snd_cwnd_stamp = tcp_time_stamp; 2541 tp->snd_ssthresh = tcp_current_ssthresh(sk); 2542 2543 icsk->icsk_mtup.search_low = icsk->icsk_mtup.probe_size; 2544 icsk->icsk_mtup.probe_size = 0; 2545 tcp_sync_mss(sk, icsk->icsk_pmtu_cookie); 2546 } 2547 2548 /* Do a simple retransmit without using the backoff mechanisms in 2549 * tcp_timer. This is used for path mtu discovery. 2550 * The socket is already locked here. 2551 */ 2552 void tcp_simple_retransmit(struct sock *sk) 2553 { 2554 const struct inet_connection_sock *icsk = inet_csk(sk); 2555 struct tcp_sock *tp = tcp_sk(sk); 2556 struct sk_buff *skb; 2557 unsigned int mss = tcp_current_mss(sk); 2558 u32 prior_lost = tp->lost_out; 2559 2560 tcp_for_write_queue(skb, sk) { 2561 if (skb == tcp_send_head(sk)) 2562 break; 2563 if (tcp_skb_seglen(skb) > mss && 2564 !(TCP_SKB_CB(skb)->sacked & TCPCB_SACKED_ACKED)) { 2565 if (TCP_SKB_CB(skb)->sacked & TCPCB_SACKED_RETRANS) { 2566 TCP_SKB_CB(skb)->sacked &= ~TCPCB_SACKED_RETRANS; 2567 tp->retrans_out -= tcp_skb_pcount(skb); 2568 } 2569 tcp_skb_mark_lost_uncond_verify(tp, skb); 2570 } 2571 } 2572 2573 tcp_clear_retrans_hints_partial(tp); 2574 2575 if (prior_lost == tp->lost_out) 2576 return; 2577 2578 if (tcp_is_reno(tp)) 2579 tcp_limit_reno_sacked(tp); 2580 2581 tcp_verify_left_out(tp); 2582 2583 /* Don't muck with the congestion window here. 2584 * Reason is that we do not increase amount of _data_ 2585 * in network, but units changed and effective 2586 * cwnd/ssthresh really reduced now. 2587 */ 2588 if (icsk->icsk_ca_state != TCP_CA_Loss) { 2589 tp->high_seq = tp->snd_nxt; 2590 tp->snd_ssthresh = tcp_current_ssthresh(sk); 2591 tp->prior_ssthresh = 0; 2592 tp->undo_marker = 0; 2593 tcp_set_ca_state(sk, TCP_CA_Loss); 2594 } 2595 tcp_xmit_retransmit_queue(sk); 2596 } 2597 EXPORT_SYMBOL(tcp_simple_retransmit); 2598 2599 static void tcp_enter_recovery(struct sock *sk, bool ece_ack) 2600 { 2601 struct tcp_sock *tp = tcp_sk(sk); 2602 int mib_idx; 2603 2604 if (tcp_is_reno(tp)) 2605 mib_idx = LINUX_MIB_TCPRENORECOVERY; 2606 else 2607 mib_idx = LINUX_MIB_TCPSACKRECOVERY; 2608 2609 NET_INC_STATS_BH(sock_net(sk), mib_idx); 2610 2611 tp->prior_ssthresh = 0; 2612 tp->undo_marker = tp->snd_una; 2613 tp->undo_retrans = tp->retrans_out; 2614 2615 if (inet_csk(sk)->icsk_ca_state < TCP_CA_CWR) { 2616 if (!ece_ack) 2617 tp->prior_ssthresh = tcp_current_ssthresh(sk); 2618 tcp_init_cwnd_reduction(sk, true); 2619 } 2620 tcp_set_ca_state(sk, TCP_CA_Recovery); 2621 } 2622 2623 /* Process an ACK in CA_Loss state. Move to CA_Open if lost data are 2624 * recovered or spurious. Otherwise retransmits more on partial ACKs. 2625 */ 2626 static void tcp_process_loss(struct sock *sk, int flag, bool is_dupack) 2627 { 2628 struct inet_connection_sock *icsk = inet_csk(sk); 2629 struct tcp_sock *tp = tcp_sk(sk); 2630 bool recovered = !before(tp->snd_una, tp->high_seq); 2631 2632 if (tp->frto) { /* F-RTO RFC5682 sec 3.1 (sack enhanced version). */ 2633 if (flag & FLAG_ORIG_SACK_ACKED) { 2634 /* Step 3.b. A timeout is spurious if not all data are 2635 * lost, i.e., never-retransmitted data are (s)acked. 2636 */ 2637 tcp_try_undo_loss(sk, true); 2638 return; 2639 } 2640 if (after(tp->snd_nxt, tp->high_seq) && 2641 (flag & FLAG_DATA_SACKED || is_dupack)) { 2642 tp->frto = 0; /* Loss was real: 2nd part of step 3.a */ 2643 } else if (flag & FLAG_SND_UNA_ADVANCED && !recovered) { 2644 tp->high_seq = tp->snd_nxt; 2645 __tcp_push_pending_frames(sk, tcp_current_mss(sk), 2646 TCP_NAGLE_OFF); 2647 if (after(tp->snd_nxt, tp->high_seq)) 2648 return; /* Step 2.b */ 2649 tp->frto = 0; 2650 } 2651 } 2652 2653 if (recovered) { 2654 /* F-RTO RFC5682 sec 3.1 step 2.a and 1st part of step 3.a */ 2655 icsk->icsk_retransmits = 0; 2656 tcp_try_undo_recovery(sk); 2657 return; 2658 } 2659 if (flag & FLAG_DATA_ACKED) 2660 icsk->icsk_retransmits = 0; 2661 if (tcp_is_reno(tp)) { 2662 /* A Reno DUPACK means new data in F-RTO step 2.b above are 2663 * delivered. Lower inflight to clock out (re)tranmissions. 2664 */ 2665 if (after(tp->snd_nxt, tp->high_seq) && is_dupack) 2666 tcp_add_reno_sack(sk); 2667 else if (flag & FLAG_SND_UNA_ADVANCED) 2668 tcp_reset_reno_sack(tp); 2669 } 2670 if (tcp_try_undo_loss(sk, false)) 2671 return; 2672 tcp_xmit_retransmit_queue(sk); 2673 } 2674 2675 /* Undo during fast recovery after partial ACK. */ 2676 static bool tcp_try_undo_partial(struct sock *sk, const int acked, 2677 const int prior_unsacked) 2678 { 2679 struct tcp_sock *tp = tcp_sk(sk); 2680 2681 if (tp->undo_marker && tcp_packet_delayed(tp)) { 2682 /* Plain luck! Hole if filled with delayed 2683 * packet, rather than with a retransmit. 2684 */ 2685 tcp_update_reordering(sk, tcp_fackets_out(tp) + acked, 1); 2686 2687 /* We are getting evidence that the reordering degree is higher 2688 * than we realized. If there are no retransmits out then we 2689 * can undo. Otherwise we clock out new packets but do not 2690 * mark more packets lost or retransmit more. 2691 */ 2692 if (tp->retrans_out) { 2693 tcp_cwnd_reduction(sk, prior_unsacked, 0); 2694 return true; 2695 } 2696 2697 if (!tcp_any_retrans_done(sk)) 2698 tp->retrans_stamp = 0; 2699 2700 DBGUNDO(sk, "partial recovery"); 2701 tcp_undo_cwnd_reduction(sk, true); 2702 NET_INC_STATS_BH(sock_net(sk), LINUX_MIB_TCPPARTIALUNDO); 2703 tcp_try_keep_open(sk); 2704 return true; 2705 } 2706 return false; 2707 } 2708 2709 /* Process an event, which can update packets-in-flight not trivially. 2710 * Main goal of this function is to calculate new estimate for left_out, 2711 * taking into account both packets sitting in receiver's buffer and 2712 * packets lost by network. 2713 * 2714 * Besides that it does CWND reduction, when packet loss is detected 2715 * and changes state of machine. 2716 * 2717 * It does _not_ decide what to send, it is made in function 2718 * tcp_xmit_retransmit_queue(). 2719 */ 2720 static void tcp_fastretrans_alert(struct sock *sk, const int acked, 2721 const int prior_unsacked, 2722 bool is_dupack, int flag) 2723 { 2724 struct inet_connection_sock *icsk = inet_csk(sk); 2725 struct tcp_sock *tp = tcp_sk(sk); 2726 bool do_lost = is_dupack || ((flag & FLAG_DATA_SACKED) && 2727 (tcp_fackets_out(tp) > tp->reordering)); 2728 int fast_rexmit = 0; 2729 2730 if (WARN_ON(!tp->packets_out && tp->sacked_out)) 2731 tp->sacked_out = 0; 2732 if (WARN_ON(!tp->sacked_out && tp->fackets_out)) 2733 tp->fackets_out = 0; 2734 2735 /* Now state machine starts. 2736 * A. ECE, hence prohibit cwnd undoing, the reduction is required. */ 2737 if (flag & FLAG_ECE) 2738 tp->prior_ssthresh = 0; 2739 2740 /* B. In all the states check for reneging SACKs. */ 2741 if (tcp_check_sack_reneging(sk, flag)) 2742 return; 2743 2744 /* C. Check consistency of the current state. */ 2745 tcp_verify_left_out(tp); 2746 2747 /* D. Check state exit conditions. State can be terminated 2748 * when high_seq is ACKed. */ 2749 if (icsk->icsk_ca_state == TCP_CA_Open) { 2750 WARN_ON(tp->retrans_out != 0); 2751 tp->retrans_stamp = 0; 2752 } else if (!before(tp->snd_una, tp->high_seq)) { 2753 switch (icsk->icsk_ca_state) { 2754 case TCP_CA_CWR: 2755 /* CWR is to be held something *above* high_seq 2756 * is ACKed for CWR bit to reach receiver. */ 2757 if (tp->snd_una != tp->high_seq) { 2758 tcp_end_cwnd_reduction(sk); 2759 tcp_set_ca_state(sk, TCP_CA_Open); 2760 } 2761 break; 2762 2763 case TCP_CA_Recovery: 2764 if (tcp_is_reno(tp)) 2765 tcp_reset_reno_sack(tp); 2766 if (tcp_try_undo_recovery(sk)) 2767 return; 2768 tcp_end_cwnd_reduction(sk); 2769 break; 2770 } 2771 } 2772 2773 /* E. Process state. */ 2774 switch (icsk->icsk_ca_state) { 2775 case TCP_CA_Recovery: 2776 if (!(flag & FLAG_SND_UNA_ADVANCED)) { 2777 if (tcp_is_reno(tp) && is_dupack) 2778 tcp_add_reno_sack(sk); 2779 } else { 2780 if (tcp_try_undo_partial(sk, acked, prior_unsacked)) 2781 return; 2782 /* Partial ACK arrived. Force fast retransmit. */ 2783 do_lost = tcp_is_reno(tp) || 2784 tcp_fackets_out(tp) > tp->reordering; 2785 } 2786 if (tcp_try_undo_dsack(sk)) { 2787 tcp_try_keep_open(sk); 2788 return; 2789 } 2790 break; 2791 case TCP_CA_Loss: 2792 tcp_process_loss(sk, flag, is_dupack); 2793 if (icsk->icsk_ca_state != TCP_CA_Open) 2794 return; 2795 /* Fall through to processing in Open state. */ 2796 default: 2797 if (tcp_is_reno(tp)) { 2798 if (flag & FLAG_SND_UNA_ADVANCED) 2799 tcp_reset_reno_sack(tp); 2800 if (is_dupack) 2801 tcp_add_reno_sack(sk); 2802 } 2803 2804 if (icsk->icsk_ca_state <= TCP_CA_Disorder) 2805 tcp_try_undo_dsack(sk); 2806 2807 if (!tcp_time_to_recover(sk, flag)) { 2808 tcp_try_to_open(sk, flag, prior_unsacked); 2809 return; 2810 } 2811 2812 /* MTU probe failure: don't reduce cwnd */ 2813 if (icsk->icsk_ca_state < TCP_CA_CWR && 2814 icsk->icsk_mtup.probe_size && 2815 tp->snd_una == tp->mtu_probe.probe_seq_start) { 2816 tcp_mtup_probe_failed(sk); 2817 /* Restores the reduction we did in tcp_mtup_probe() */ 2818 tp->snd_cwnd++; 2819 tcp_simple_retransmit(sk); 2820 return; 2821 } 2822 2823 /* Otherwise enter Recovery state */ 2824 tcp_enter_recovery(sk, (flag & FLAG_ECE)); 2825 fast_rexmit = 1; 2826 } 2827 2828 if (do_lost) 2829 tcp_update_scoreboard(sk, fast_rexmit); 2830 tcp_cwnd_reduction(sk, prior_unsacked, fast_rexmit); 2831 tcp_xmit_retransmit_queue(sk); 2832 } 2833 2834 static inline bool tcp_ack_update_rtt(struct sock *sk, const int flag, 2835 s32 seq_rtt, s32 sack_rtt) 2836 { 2837 const struct tcp_sock *tp = tcp_sk(sk); 2838 2839 /* Prefer RTT measured from ACK's timing to TS-ECR. This is because 2840 * broken middle-boxes or peers may corrupt TS-ECR fields. But 2841 * Karn's algorithm forbids taking RTT if some retransmitted data 2842 * is acked (RFC6298). 2843 */ 2844 if (flag & FLAG_RETRANS_DATA_ACKED) 2845 seq_rtt = -1; 2846 2847 if (seq_rtt < 0) 2848 seq_rtt = sack_rtt; 2849 2850 /* RTTM Rule: A TSecr value received in a segment is used to 2851 * update the averaged RTT measurement only if the segment 2852 * acknowledges some new data, i.e., only if it advances the 2853 * left edge of the send window. 2854 * See draft-ietf-tcplw-high-performance-00, section 3.3. 2855 */ 2856 if (seq_rtt < 0 && tp->rx_opt.saw_tstamp && tp->rx_opt.rcv_tsecr) 2857 seq_rtt = tcp_time_stamp - tp->rx_opt.rcv_tsecr; 2858 2859 if (seq_rtt < 0) 2860 return false; 2861 2862 tcp_rtt_estimator(sk, seq_rtt); 2863 tcp_set_rto(sk); 2864 2865 /* RFC6298: only reset backoff on valid RTT measurement. */ 2866 inet_csk(sk)->icsk_backoff = 0; 2867 return true; 2868 } 2869 2870 /* Compute time elapsed between (last) SYNACK and the ACK completing 3WHS. */ 2871 static void tcp_synack_rtt_meas(struct sock *sk, struct request_sock *req) 2872 { 2873 struct tcp_sock *tp = tcp_sk(sk); 2874 s32 seq_rtt = -1; 2875 2876 if (tp->lsndtime && !tp->total_retrans) 2877 seq_rtt = tcp_time_stamp - tp->lsndtime; 2878 tcp_ack_update_rtt(sk, FLAG_SYN_ACKED, seq_rtt, -1); 2879 } 2880 2881 static void tcp_cong_avoid(struct sock *sk, u32 ack, u32 in_flight) 2882 { 2883 const struct inet_connection_sock *icsk = inet_csk(sk); 2884 icsk->icsk_ca_ops->cong_avoid(sk, ack, in_flight); 2885 tcp_sk(sk)->snd_cwnd_stamp = tcp_time_stamp; 2886 } 2887 2888 /* Restart timer after forward progress on connection. 2889 * RFC2988 recommends to restart timer to now+rto. 2890 */ 2891 void tcp_rearm_rto(struct sock *sk) 2892 { 2893 const struct inet_connection_sock *icsk = inet_csk(sk); 2894 struct tcp_sock *tp = tcp_sk(sk); 2895 2896 /* If the retrans timer is currently being used by Fast Open 2897 * for SYN-ACK retrans purpose, stay put. 2898 */ 2899 if (tp->fastopen_rsk) 2900 return; 2901 2902 if (!tp->packets_out) { 2903 inet_csk_clear_xmit_timer(sk, ICSK_TIME_RETRANS); 2904 } else { 2905 u32 rto = inet_csk(sk)->icsk_rto; 2906 /* Offset the time elapsed after installing regular RTO */ 2907 if (icsk->icsk_pending == ICSK_TIME_EARLY_RETRANS || 2908 icsk->icsk_pending == ICSK_TIME_LOSS_PROBE) { 2909 struct sk_buff *skb = tcp_write_queue_head(sk); 2910 const u32 rto_time_stamp = TCP_SKB_CB(skb)->when + rto; 2911 s32 delta = (s32)(rto_time_stamp - tcp_time_stamp); 2912 /* delta may not be positive if the socket is locked 2913 * when the retrans timer fires and is rescheduled. 2914 */ 2915 if (delta > 0) 2916 rto = delta; 2917 } 2918 inet_csk_reset_xmit_timer(sk, ICSK_TIME_RETRANS, rto, 2919 TCP_RTO_MAX); 2920 } 2921 } 2922 2923 /* This function is called when the delayed ER timer fires. TCP enters 2924 * fast recovery and performs fast-retransmit. 2925 */ 2926 void tcp_resume_early_retransmit(struct sock *sk) 2927 { 2928 struct tcp_sock *tp = tcp_sk(sk); 2929 2930 tcp_rearm_rto(sk); 2931 2932 /* Stop if ER is disabled after the delayed ER timer is scheduled */ 2933 if (!tp->do_early_retrans) 2934 return; 2935 2936 tcp_enter_recovery(sk, false); 2937 tcp_update_scoreboard(sk, 1); 2938 tcp_xmit_retransmit_queue(sk); 2939 } 2940 2941 /* If we get here, the whole TSO packet has not been acked. */ 2942 static u32 tcp_tso_acked(struct sock *sk, struct sk_buff *skb) 2943 { 2944 struct tcp_sock *tp = tcp_sk(sk); 2945 u32 packets_acked; 2946 2947 BUG_ON(!after(TCP_SKB_CB(skb)->end_seq, tp->snd_una)); 2948 2949 packets_acked = tcp_skb_pcount(skb); 2950 if (tcp_trim_head(sk, skb, tp->snd_una - TCP_SKB_CB(skb)->seq)) 2951 return 0; 2952 packets_acked -= tcp_skb_pcount(skb); 2953 2954 if (packets_acked) { 2955 BUG_ON(tcp_skb_pcount(skb) == 0); 2956 BUG_ON(!before(TCP_SKB_CB(skb)->seq, TCP_SKB_CB(skb)->end_seq)); 2957 } 2958 2959 return packets_acked; 2960 } 2961 2962 /* Remove acknowledged frames from the retransmission queue. If our packet 2963 * is before the ack sequence we can discard it as it's confirmed to have 2964 * arrived at the other end. 2965 */ 2966 static int tcp_clean_rtx_queue(struct sock *sk, int prior_fackets, 2967 u32 prior_snd_una, s32 sack_rtt) 2968 { 2969 struct tcp_sock *tp = tcp_sk(sk); 2970 const struct inet_connection_sock *icsk = inet_csk(sk); 2971 struct sk_buff *skb; 2972 u32 now = tcp_time_stamp; 2973 int fully_acked = true; 2974 int flag = 0; 2975 u32 pkts_acked = 0; 2976 u32 reord = tp->packets_out; 2977 u32 prior_sacked = tp->sacked_out; 2978 s32 seq_rtt = -1; 2979 s32 ca_seq_rtt = -1; 2980 ktime_t last_ackt = net_invalid_timestamp(); 2981 2982 while ((skb = tcp_write_queue_head(sk)) && skb != tcp_send_head(sk)) { 2983 struct tcp_skb_cb *scb = TCP_SKB_CB(skb); 2984 u32 acked_pcount; 2985 u8 sacked = scb->sacked; 2986 2987 /* Determine how many packets and what bytes were acked, tso and else */ 2988 if (after(scb->end_seq, tp->snd_una)) { 2989 if (tcp_skb_pcount(skb) == 1 || 2990 !after(tp->snd_una, scb->seq)) 2991 break; 2992 2993 acked_pcount = tcp_tso_acked(sk, skb); 2994 if (!acked_pcount) 2995 break; 2996 2997 fully_acked = false; 2998 } else { 2999 acked_pcount = tcp_skb_pcount(skb); 3000 } 3001 3002 if (sacked & TCPCB_RETRANS) { 3003 if (sacked & TCPCB_SACKED_RETRANS) 3004 tp->retrans_out -= acked_pcount; 3005 flag |= FLAG_RETRANS_DATA_ACKED; 3006 } else { 3007 ca_seq_rtt = now - scb->when; 3008 last_ackt = skb->tstamp; 3009 if (seq_rtt < 0) { 3010 seq_rtt = ca_seq_rtt; 3011 } 3012 if (!(sacked & TCPCB_SACKED_ACKED)) 3013 reord = min(pkts_acked, reord); 3014 if (!after(scb->end_seq, tp->high_seq)) 3015 flag |= FLAG_ORIG_SACK_ACKED; 3016 } 3017 3018 if (sacked & TCPCB_SACKED_ACKED) 3019 tp->sacked_out -= acked_pcount; 3020 if (sacked & TCPCB_LOST) 3021 tp->lost_out -= acked_pcount; 3022 3023 tp->packets_out -= acked_pcount; 3024 pkts_acked += acked_pcount; 3025 3026 /* Initial outgoing SYN's get put onto the write_queue 3027 * just like anything else we transmit. It is not 3028 * true data, and if we misinform our callers that 3029 * this ACK acks real data, we will erroneously exit 3030 * connection startup slow start one packet too 3031 * quickly. This is severely frowned upon behavior. 3032 */ 3033 if (!(scb->tcp_flags & TCPHDR_SYN)) { 3034 flag |= FLAG_DATA_ACKED; 3035 } else { 3036 flag |= FLAG_SYN_ACKED; 3037 tp->retrans_stamp = 0; 3038 } 3039 3040 if (!fully_acked) 3041 break; 3042 3043 tcp_unlink_write_queue(skb, sk); 3044 sk_wmem_free_skb(sk, skb); 3045 if (skb == tp->retransmit_skb_hint) 3046 tp->retransmit_skb_hint = NULL; 3047 if (skb == tp->lost_skb_hint) 3048 tp->lost_skb_hint = NULL; 3049 } 3050 3051 if (likely(between(tp->snd_up, prior_snd_una, tp->snd_una))) 3052 tp->snd_up = tp->snd_una; 3053 3054 if (skb && (TCP_SKB_CB(skb)->sacked & TCPCB_SACKED_ACKED)) 3055 flag |= FLAG_SACK_RENEGING; 3056 3057 if (tcp_ack_update_rtt(sk, flag, seq_rtt, sack_rtt) || 3058 (flag & FLAG_ACKED)) 3059 tcp_rearm_rto(sk); 3060 3061 if (flag & FLAG_ACKED) { 3062 const struct tcp_congestion_ops *ca_ops 3063 = inet_csk(sk)->icsk_ca_ops; 3064 3065 if (unlikely(icsk->icsk_mtup.probe_size && 3066 !after(tp->mtu_probe.probe_seq_end, tp->snd_una))) { 3067 tcp_mtup_probe_success(sk); 3068 } 3069 3070 if (tcp_is_reno(tp)) { 3071 tcp_remove_reno_sacks(sk, pkts_acked); 3072 } else { 3073 int delta; 3074 3075 /* Non-retransmitted hole got filled? That's reordering */ 3076 if (reord < prior_fackets) 3077 tcp_update_reordering(sk, tp->fackets_out - reord, 0); 3078 3079 delta = tcp_is_fack(tp) ? pkts_acked : 3080 prior_sacked - tp->sacked_out; 3081 tp->lost_cnt_hint -= min(tp->lost_cnt_hint, delta); 3082 } 3083 3084 tp->fackets_out -= min(pkts_acked, tp->fackets_out); 3085 3086 if (ca_ops->pkts_acked) { 3087 s32 rtt_us = -1; 3088 3089 /* Is the ACK triggering packet unambiguous? */ 3090 if (!(flag & FLAG_RETRANS_DATA_ACKED)) { 3091 /* High resolution needed and available? */ 3092 if (ca_ops->flags & TCP_CONG_RTT_STAMP && 3093 !ktime_equal(last_ackt, 3094 net_invalid_timestamp())) 3095 rtt_us = ktime_us_delta(ktime_get_real(), 3096 last_ackt); 3097 else if (ca_seq_rtt >= 0) 3098 rtt_us = jiffies_to_usecs(ca_seq_rtt); 3099 } 3100 3101 ca_ops->pkts_acked(sk, pkts_acked, rtt_us); 3102 } 3103 } 3104 3105 #if FASTRETRANS_DEBUG > 0 3106 WARN_ON((int)tp->sacked_out < 0); 3107 WARN_ON((int)tp->lost_out < 0); 3108 WARN_ON((int)tp->retrans_out < 0); 3109 if (!tp->packets_out && tcp_is_sack(tp)) { 3110 icsk = inet_csk(sk); 3111 if (tp->lost_out) { 3112 pr_debug("Leak l=%u %d\n", 3113 tp->lost_out, icsk->icsk_ca_state); 3114 tp->lost_out = 0; 3115 } 3116 if (tp->sacked_out) { 3117 pr_debug("Leak s=%u %d\n", 3118 tp->sacked_out, icsk->icsk_ca_state); 3119 tp->sacked_out = 0; 3120 } 3121 if (tp->retrans_out) { 3122 pr_debug("Leak r=%u %d\n", 3123 tp->retrans_out, icsk->icsk_ca_state); 3124 tp->retrans_out = 0; 3125 } 3126 } 3127 #endif 3128 return flag; 3129 } 3130 3131 static void tcp_ack_probe(struct sock *sk) 3132 { 3133 const struct tcp_sock *tp = tcp_sk(sk); 3134 struct inet_connection_sock *icsk = inet_csk(sk); 3135 3136 /* Was it a usable window open? */ 3137 3138 if (!after(TCP_SKB_CB(tcp_send_head(sk))->end_seq, tcp_wnd_end(tp))) { 3139 icsk->icsk_backoff = 0; 3140 inet_csk_clear_xmit_timer(sk, ICSK_TIME_PROBE0); 3141 /* Socket must be waked up by subsequent tcp_data_snd_check(). 3142 * This function is not for random using! 3143 */ 3144 } else { 3145 inet_csk_reset_xmit_timer(sk, ICSK_TIME_PROBE0, 3146 min(icsk->icsk_rto << icsk->icsk_backoff, TCP_RTO_MAX), 3147 TCP_RTO_MAX); 3148 } 3149 } 3150 3151 static inline bool tcp_ack_is_dubious(const struct sock *sk, const int flag) 3152 { 3153 return !(flag & FLAG_NOT_DUP) || (flag & FLAG_CA_ALERT) || 3154 inet_csk(sk)->icsk_ca_state != TCP_CA_Open; 3155 } 3156 3157 /* Decide wheather to run the increase function of congestion control. */ 3158 static inline bool tcp_may_raise_cwnd(const struct sock *sk, const int flag) 3159 { 3160 if (tcp_in_cwnd_reduction(sk)) 3161 return false; 3162 3163 /* If reordering is high then always grow cwnd whenever data is 3164 * delivered regardless of its ordering. Otherwise stay conservative 3165 * and only grow cwnd on in-order delivery (RFC5681). A stretched ACK w/ 3166 * new SACK or ECE mark may first advance cwnd here and later reduce 3167 * cwnd in tcp_fastretrans_alert() based on more states. 3168 */ 3169 if (tcp_sk(sk)->reordering > sysctl_tcp_reordering) 3170 return flag & FLAG_FORWARD_PROGRESS; 3171 3172 return flag & FLAG_DATA_ACKED; 3173 } 3174 3175 /* Check that window update is acceptable. 3176 * The function assumes that snd_una<=ack<=snd_next. 3177 */ 3178 static inline bool tcp_may_update_window(const struct tcp_sock *tp, 3179 const u32 ack, const u32 ack_seq, 3180 const u32 nwin) 3181 { 3182 return after(ack, tp->snd_una) || 3183 after(ack_seq, tp->snd_wl1) || 3184 (ack_seq == tp->snd_wl1 && nwin > tp->snd_wnd); 3185 } 3186 3187 /* Update our send window. 3188 * 3189 * Window update algorithm, described in RFC793/RFC1122 (used in linux-2.2 3190 * and in FreeBSD. NetBSD's one is even worse.) is wrong. 3191 */ 3192 static int tcp_ack_update_window(struct sock *sk, const struct sk_buff *skb, u32 ack, 3193 u32 ack_seq) 3194 { 3195 struct tcp_sock *tp = tcp_sk(sk); 3196 int flag = 0; 3197 u32 nwin = ntohs(tcp_hdr(skb)->window); 3198 3199 if (likely(!tcp_hdr(skb)->syn)) 3200 nwin <<= tp->rx_opt.snd_wscale; 3201 3202 if (tcp_may_update_window(tp, ack, ack_seq, nwin)) { 3203 flag |= FLAG_WIN_UPDATE; 3204 tcp_update_wl(tp, ack_seq); 3205 3206 if (tp->snd_wnd != nwin) { 3207 tp->snd_wnd = nwin; 3208 3209 /* Note, it is the only place, where 3210 * fast path is recovered for sending TCP. 3211 */ 3212 tp->pred_flags = 0; 3213 tcp_fast_path_check(sk); 3214 3215 if (nwin > tp->max_window) { 3216 tp->max_window = nwin; 3217 tcp_sync_mss(sk, inet_csk(sk)->icsk_pmtu_cookie); 3218 } 3219 } 3220 } 3221 3222 tp->snd_una = ack; 3223 3224 return flag; 3225 } 3226 3227 /* RFC 5961 7 [ACK Throttling] */ 3228 static void tcp_send_challenge_ack(struct sock *sk) 3229 { 3230 /* unprotected vars, we dont care of overwrites */ 3231 static u32 challenge_timestamp; 3232 static unsigned int challenge_count; 3233 u32 now = jiffies / HZ; 3234 3235 if (now != challenge_timestamp) { 3236 challenge_timestamp = now; 3237 challenge_count = 0; 3238 } 3239 if (++challenge_count <= sysctl_tcp_challenge_ack_limit) { 3240 NET_INC_STATS_BH(sock_net(sk), LINUX_MIB_TCPCHALLENGEACK); 3241 tcp_send_ack(sk); 3242 } 3243 } 3244 3245 static void tcp_store_ts_recent(struct tcp_sock *tp) 3246 { 3247 tp->rx_opt.ts_recent = tp->rx_opt.rcv_tsval; 3248 tp->rx_opt.ts_recent_stamp = get_seconds(); 3249 } 3250 3251 static void tcp_replace_ts_recent(struct tcp_sock *tp, u32 seq) 3252 { 3253 if (tp->rx_opt.saw_tstamp && !after(seq, tp->rcv_wup)) { 3254 /* PAWS bug workaround wrt. ACK frames, the PAWS discard 3255 * extra check below makes sure this can only happen 3256 * for pure ACK frames. -DaveM 3257 * 3258 * Not only, also it occurs for expired timestamps. 3259 */ 3260 3261 if (tcp_paws_check(&tp->rx_opt, 0)) 3262 tcp_store_ts_recent(tp); 3263 } 3264 } 3265 3266 /* This routine deals with acks during a TLP episode. 3267 * Ref: loss detection algorithm in draft-dukkipati-tcpm-tcp-loss-probe. 3268 */ 3269 static void tcp_process_tlp_ack(struct sock *sk, u32 ack, int flag) 3270 { 3271 struct tcp_sock *tp = tcp_sk(sk); 3272 bool is_tlp_dupack = (ack == tp->tlp_high_seq) && 3273 !(flag & (FLAG_SND_UNA_ADVANCED | 3274 FLAG_NOT_DUP | FLAG_DATA_SACKED)); 3275 3276 /* Mark the end of TLP episode on receiving TLP dupack or when 3277 * ack is after tlp_high_seq. 3278 */ 3279 if (is_tlp_dupack) { 3280 tp->tlp_high_seq = 0; 3281 return; 3282 } 3283 3284 if (after(ack, tp->tlp_high_seq)) { 3285 tp->tlp_high_seq = 0; 3286 /* Don't reduce cwnd if DSACK arrives for TLP retrans. */ 3287 if (!(flag & FLAG_DSACKING_ACK)) { 3288 tcp_init_cwnd_reduction(sk, true); 3289 tcp_set_ca_state(sk, TCP_CA_CWR); 3290 tcp_end_cwnd_reduction(sk); 3291 tcp_set_ca_state(sk, TCP_CA_Open); 3292 NET_INC_STATS_BH(sock_net(sk), 3293 LINUX_MIB_TCPLOSSPROBERECOVERY); 3294 } 3295 } 3296 } 3297 3298 /* This routine deals with incoming acks, but not outgoing ones. */ 3299 static int tcp_ack(struct sock *sk, const struct sk_buff *skb, int flag) 3300 { 3301 struct inet_connection_sock *icsk = inet_csk(sk); 3302 struct tcp_sock *tp = tcp_sk(sk); 3303 u32 prior_snd_una = tp->snd_una; 3304 u32 ack_seq = TCP_SKB_CB(skb)->seq; 3305 u32 ack = TCP_SKB_CB(skb)->ack_seq; 3306 bool is_dupack = false; 3307 u32 prior_in_flight, prior_cwnd = tp->snd_cwnd, prior_rtt = tp->srtt; 3308 u32 prior_fackets; 3309 int prior_packets = tp->packets_out; 3310 const int prior_unsacked = tp->packets_out - tp->sacked_out; 3311 int acked = 0; /* Number of packets newly acked */ 3312 s32 sack_rtt = -1; 3313 3314 /* If the ack is older than previous acks 3315 * then we can probably ignore it. 3316 */ 3317 if (before(ack, prior_snd_una)) { 3318 /* RFC 5961 5.2 [Blind Data Injection Attack].[Mitigation] */ 3319 if (before(ack, prior_snd_una - tp->max_window)) { 3320 tcp_send_challenge_ack(sk); 3321 return -1; 3322 } 3323 goto old_ack; 3324 } 3325 3326 /* If the ack includes data we haven't sent yet, discard 3327 * this segment (RFC793 Section 3.9). 3328 */ 3329 if (after(ack, tp->snd_nxt)) 3330 goto invalid_ack; 3331 3332 if (icsk->icsk_pending == ICSK_TIME_EARLY_RETRANS || 3333 icsk->icsk_pending == ICSK_TIME_LOSS_PROBE) 3334 tcp_rearm_rto(sk); 3335 3336 if (after(ack, prior_snd_una)) 3337 flag |= FLAG_SND_UNA_ADVANCED; 3338 3339 prior_fackets = tp->fackets_out; 3340 prior_in_flight = tcp_packets_in_flight(tp); 3341 3342 /* ts_recent update must be made after we are sure that the packet 3343 * is in window. 3344 */ 3345 if (flag & FLAG_UPDATE_TS_RECENT) 3346 tcp_replace_ts_recent(tp, TCP_SKB_CB(skb)->seq); 3347 3348 if (!(flag & FLAG_SLOWPATH) && after(ack, prior_snd_una)) { 3349 /* Window is constant, pure forward advance. 3350 * No more checks are required. 3351 * Note, we use the fact that SND.UNA>=SND.WL2. 3352 */ 3353 tcp_update_wl(tp, ack_seq); 3354 tp->snd_una = ack; 3355 flag |= FLAG_WIN_UPDATE; 3356 3357 tcp_ca_event(sk, CA_EVENT_FAST_ACK); 3358 3359 NET_INC_STATS_BH(sock_net(sk), LINUX_MIB_TCPHPACKS); 3360 } else { 3361 if (ack_seq != TCP_SKB_CB(skb)->end_seq) 3362 flag |= FLAG_DATA; 3363 else 3364 NET_INC_STATS_BH(sock_net(sk), LINUX_MIB_TCPPUREACKS); 3365 3366 flag |= tcp_ack_update_window(sk, skb, ack, ack_seq); 3367 3368 if (TCP_SKB_CB(skb)->sacked) 3369 flag |= tcp_sacktag_write_queue(sk, skb, prior_snd_una, 3370 &sack_rtt); 3371 3372 if (TCP_ECN_rcv_ecn_echo(tp, tcp_hdr(skb))) 3373 flag |= FLAG_ECE; 3374 3375 tcp_ca_event(sk, CA_EVENT_SLOW_ACK); 3376 } 3377 3378 /* We passed data and got it acked, remove any soft error 3379 * log. Something worked... 3380 */ 3381 sk->sk_err_soft = 0; 3382 icsk->icsk_probes_out = 0; 3383 tp->rcv_tstamp = tcp_time_stamp; 3384 if (!prior_packets) 3385 goto no_queue; 3386 3387 /* See if we can take anything off of the retransmit queue. */ 3388 acked = tp->packets_out; 3389 flag |= tcp_clean_rtx_queue(sk, prior_fackets, prior_snd_una, sack_rtt); 3390 acked -= tp->packets_out; 3391 3392 /* Advance cwnd if state allows */ 3393 if (tcp_may_raise_cwnd(sk, flag)) 3394 tcp_cong_avoid(sk, ack, prior_in_flight); 3395 3396 if (tcp_ack_is_dubious(sk, flag)) { 3397 is_dupack = !(flag & (FLAG_SND_UNA_ADVANCED | FLAG_NOT_DUP)); 3398 tcp_fastretrans_alert(sk, acked, prior_unsacked, 3399 is_dupack, flag); 3400 } 3401 if (tp->tlp_high_seq) 3402 tcp_process_tlp_ack(sk, ack, flag); 3403 3404 if ((flag & FLAG_FORWARD_PROGRESS) || !(flag & FLAG_NOT_DUP)) { 3405 struct dst_entry *dst = __sk_dst_get(sk); 3406 if (dst) 3407 dst_confirm(dst); 3408 } 3409 3410 if (icsk->icsk_pending == ICSK_TIME_RETRANS) 3411 tcp_schedule_loss_probe(sk); 3412 if (tp->srtt != prior_rtt || tp->snd_cwnd != prior_cwnd) 3413 tcp_update_pacing_rate(sk); 3414 return 1; 3415 3416 no_queue: 3417 /* If data was DSACKed, see if we can undo a cwnd reduction. */ 3418 if (flag & FLAG_DSACKING_ACK) 3419 tcp_fastretrans_alert(sk, acked, prior_unsacked, 3420 is_dupack, flag); 3421 /* If this ack opens up a zero window, clear backoff. It was 3422 * being used to time the probes, and is probably far higher than 3423 * it needs to be for normal retransmission. 3424 */ 3425 if (tcp_send_head(sk)) 3426 tcp_ack_probe(sk); 3427 3428 if (tp->tlp_high_seq) 3429 tcp_process_tlp_ack(sk, ack, flag); 3430 return 1; 3431 3432 invalid_ack: 3433 SOCK_DEBUG(sk, "Ack %u after %u:%u\n", ack, tp->snd_una, tp->snd_nxt); 3434 return -1; 3435 3436 old_ack: 3437 /* If data was SACKed, tag it and see if we should send more data. 3438 * If data was DSACKed, see if we can undo a cwnd reduction. 3439 */ 3440 if (TCP_SKB_CB(skb)->sacked) { 3441 flag |= tcp_sacktag_write_queue(sk, skb, prior_snd_una, 3442 &sack_rtt); 3443 tcp_fastretrans_alert(sk, acked, prior_unsacked, 3444 is_dupack, flag); 3445 } 3446 3447 SOCK_DEBUG(sk, "Ack %u before %u:%u\n", ack, tp->snd_una, tp->snd_nxt); 3448 return 0; 3449 } 3450 3451 /* Look for tcp options. Normally only called on SYN and SYNACK packets. 3452 * But, this can also be called on packets in the established flow when 3453 * the fast version below fails. 3454 */ 3455 void tcp_parse_options(const struct sk_buff *skb, 3456 struct tcp_options_received *opt_rx, int estab, 3457 struct tcp_fastopen_cookie *foc) 3458 { 3459 const unsigned char *ptr; 3460 const struct tcphdr *th = tcp_hdr(skb); 3461 int length = (th->doff * 4) - sizeof(struct tcphdr); 3462 3463 ptr = (const unsigned char *)(th + 1); 3464 opt_rx->saw_tstamp = 0; 3465 3466 while (length > 0) { 3467 int opcode = *ptr++; 3468 int opsize; 3469 3470 switch (opcode) { 3471 case TCPOPT_EOL: 3472 return; 3473 case TCPOPT_NOP: /* Ref: RFC 793 section 3.1 */ 3474 length--; 3475 continue; 3476 default: 3477 opsize = *ptr++; 3478 if (opsize < 2) /* "silly options" */ 3479 return; 3480 if (opsize > length) 3481 return; /* don't parse partial options */ 3482 switch (opcode) { 3483 case TCPOPT_MSS: 3484 if (opsize == TCPOLEN_MSS && th->syn && !estab) { 3485 u16 in_mss = get_unaligned_be16(ptr); 3486 if (in_mss) { 3487 if (opt_rx->user_mss && 3488 opt_rx->user_mss < in_mss) 3489 in_mss = opt_rx->user_mss; 3490 opt_rx->mss_clamp = in_mss; 3491 } 3492 } 3493 break; 3494 case TCPOPT_WINDOW: 3495 if (opsize == TCPOLEN_WINDOW && th->syn && 3496 !estab && sysctl_tcp_window_scaling) { 3497 __u8 snd_wscale = *(__u8 *)ptr; 3498 opt_rx->wscale_ok = 1; 3499 if (snd_wscale > 14) { 3500 net_info_ratelimited("%s: Illegal window scaling value %d >14 received\n", 3501 __func__, 3502 snd_wscale); 3503 snd_wscale = 14; 3504 } 3505 opt_rx->snd_wscale = snd_wscale; 3506 } 3507 break; 3508 case TCPOPT_TIMESTAMP: 3509 if ((opsize == TCPOLEN_TIMESTAMP) && 3510 ((estab && opt_rx->tstamp_ok) || 3511 (!estab && sysctl_tcp_timestamps))) { 3512 opt_rx->saw_tstamp = 1; 3513 opt_rx->rcv_tsval = get_unaligned_be32(ptr); 3514 opt_rx->rcv_tsecr = get_unaligned_be32(ptr + 4); 3515 } 3516 break; 3517 case TCPOPT_SACK_PERM: 3518 if (opsize == TCPOLEN_SACK_PERM && th->syn && 3519 !estab && sysctl_tcp_sack) { 3520 opt_rx->sack_ok = TCP_SACK_SEEN; 3521 tcp_sack_reset(opt_rx); 3522 } 3523 break; 3524 3525 case TCPOPT_SACK: 3526 if ((opsize >= (TCPOLEN_SACK_BASE + TCPOLEN_SACK_PERBLOCK)) && 3527 !((opsize - TCPOLEN_SACK_BASE) % TCPOLEN_SACK_PERBLOCK) && 3528 opt_rx->sack_ok) { 3529 TCP_SKB_CB(skb)->sacked = (ptr - 2) - (unsigned char *)th; 3530 } 3531 break; 3532 #ifdef CONFIG_TCP_MD5SIG 3533 case TCPOPT_MD5SIG: 3534 /* 3535 * The MD5 Hash has already been 3536 * checked (see tcp_v{4,6}_do_rcv()). 3537 */ 3538 break; 3539 #endif 3540 case TCPOPT_EXP: 3541 /* Fast Open option shares code 254 using a 3542 * 16 bits magic number. It's valid only in 3543 * SYN or SYN-ACK with an even size. 3544 */ 3545 if (opsize < TCPOLEN_EXP_FASTOPEN_BASE || 3546 get_unaligned_be16(ptr) != TCPOPT_FASTOPEN_MAGIC || 3547 foc == NULL || !th->syn || (opsize & 1)) 3548 break; 3549 foc->len = opsize - TCPOLEN_EXP_FASTOPEN_BASE; 3550 if (foc->len >= TCP_FASTOPEN_COOKIE_MIN && 3551 foc->len <= TCP_FASTOPEN_COOKIE_MAX) 3552 memcpy(foc->val, ptr + 2, foc->len); 3553 else if (foc->len != 0) 3554 foc->len = -1; 3555 break; 3556 3557 } 3558 ptr += opsize-2; 3559 length -= opsize; 3560 } 3561 } 3562 } 3563 EXPORT_SYMBOL(tcp_parse_options); 3564 3565 static bool tcp_parse_aligned_timestamp(struct tcp_sock *tp, const struct tcphdr *th) 3566 { 3567 const __be32 *ptr = (const __be32 *)(th + 1); 3568 3569 if (*ptr == htonl((TCPOPT_NOP << 24) | (TCPOPT_NOP << 16) 3570 | (TCPOPT_TIMESTAMP << 8) | TCPOLEN_TIMESTAMP)) { 3571 tp->rx_opt.saw_tstamp = 1; 3572 ++ptr; 3573 tp->rx_opt.rcv_tsval = ntohl(*ptr); 3574 ++ptr; 3575 if (*ptr) 3576 tp->rx_opt.rcv_tsecr = ntohl(*ptr) - tp->tsoffset; 3577 else 3578 tp->rx_opt.rcv_tsecr = 0; 3579 return true; 3580 } 3581 return false; 3582 } 3583 3584 /* Fast parse options. This hopes to only see timestamps. 3585 * If it is wrong it falls back on tcp_parse_options(). 3586 */ 3587 static bool tcp_fast_parse_options(const struct sk_buff *skb, 3588 const struct tcphdr *th, struct tcp_sock *tp) 3589 { 3590 /* In the spirit of fast parsing, compare doff directly to constant 3591 * values. Because equality is used, short doff can be ignored here. 3592 */ 3593 if (th->doff == (sizeof(*th) / 4)) { 3594 tp->rx_opt.saw_tstamp = 0; 3595 return false; 3596 } else if (tp->rx_opt.tstamp_ok && 3597 th->doff == ((sizeof(*th) + TCPOLEN_TSTAMP_ALIGNED) / 4)) { 3598 if (tcp_parse_aligned_timestamp(tp, th)) 3599 return true; 3600 } 3601 3602 tcp_parse_options(skb, &tp->rx_opt, 1, NULL); 3603 if (tp->rx_opt.saw_tstamp && tp->rx_opt.rcv_tsecr) 3604 tp->rx_opt.rcv_tsecr -= tp->tsoffset; 3605 3606 return true; 3607 } 3608 3609 #ifdef CONFIG_TCP_MD5SIG 3610 /* 3611 * Parse MD5 Signature option 3612 */ 3613 const u8 *tcp_parse_md5sig_option(const struct tcphdr *th) 3614 { 3615 int length = (th->doff << 2) - sizeof(*th); 3616 const u8 *ptr = (const u8 *)(th + 1); 3617 3618 /* If the TCP option is too short, we can short cut */ 3619 if (length < TCPOLEN_MD5SIG) 3620 return NULL; 3621 3622 while (length > 0) { 3623 int opcode = *ptr++; 3624 int opsize; 3625 3626 switch(opcode) { 3627 case TCPOPT_EOL: 3628 return NULL; 3629 case TCPOPT_NOP: 3630 length--; 3631 continue; 3632 default: 3633 opsize = *ptr++; 3634 if (opsize < 2 || opsize > length) 3635 return NULL; 3636 if (opcode == TCPOPT_MD5SIG) 3637 return opsize == TCPOLEN_MD5SIG ? ptr : NULL; 3638 } 3639 ptr += opsize - 2; 3640 length -= opsize; 3641 } 3642 return NULL; 3643 } 3644 EXPORT_SYMBOL(tcp_parse_md5sig_option); 3645 #endif 3646 3647 /* Sorry, PAWS as specified is broken wrt. pure-ACKs -DaveM 3648 * 3649 * It is not fatal. If this ACK does _not_ change critical state (seqs, window) 3650 * it can pass through stack. So, the following predicate verifies that 3651 * this segment is not used for anything but congestion avoidance or 3652 * fast retransmit. Moreover, we even are able to eliminate most of such 3653 * second order effects, if we apply some small "replay" window (~RTO) 3654 * to timestamp space. 3655 * 3656 * All these measures still do not guarantee that we reject wrapped ACKs 3657 * on networks with high bandwidth, when sequence space is recycled fastly, 3658 * but it guarantees that such events will be very rare and do not affect 3659 * connection seriously. This doesn't look nice, but alas, PAWS is really 3660 * buggy extension. 3661 * 3662 * [ Later note. Even worse! It is buggy for segments _with_ data. RFC 3663 * states that events when retransmit arrives after original data are rare. 3664 * It is a blatant lie. VJ forgot about fast retransmit! 8)8) It is 3665 * the biggest problem on large power networks even with minor reordering. 3666 * OK, let's give it small replay window. If peer clock is even 1hz, it is safe 3667 * up to bandwidth of 18Gigabit/sec. 8) ] 3668 */ 3669 3670 static int tcp_disordered_ack(const struct sock *sk, const struct sk_buff *skb) 3671 { 3672 const struct tcp_sock *tp = tcp_sk(sk); 3673 const struct tcphdr *th = tcp_hdr(skb); 3674 u32 seq = TCP_SKB_CB(skb)->seq; 3675 u32 ack = TCP_SKB_CB(skb)->ack_seq; 3676 3677 return (/* 1. Pure ACK with correct sequence number. */ 3678 (th->ack && seq == TCP_SKB_CB(skb)->end_seq && seq == tp->rcv_nxt) && 3679 3680 /* 2. ... and duplicate ACK. */ 3681 ack == tp->snd_una && 3682 3683 /* 3. ... and does not update window. */ 3684 !tcp_may_update_window(tp, ack, seq, ntohs(th->window) << tp->rx_opt.snd_wscale) && 3685 3686 /* 4. ... and sits in replay window. */ 3687 (s32)(tp->rx_opt.ts_recent - tp->rx_opt.rcv_tsval) <= (inet_csk(sk)->icsk_rto * 1024) / HZ); 3688 } 3689 3690 static inline bool tcp_paws_discard(const struct sock *sk, 3691 const struct sk_buff *skb) 3692 { 3693 const struct tcp_sock *tp = tcp_sk(sk); 3694 3695 return !tcp_paws_check(&tp->rx_opt, TCP_PAWS_WINDOW) && 3696 !tcp_disordered_ack(sk, skb); 3697 } 3698 3699 /* Check segment sequence number for validity. 3700 * 3701 * Segment controls are considered valid, if the segment 3702 * fits to the window after truncation to the window. Acceptability 3703 * of data (and SYN, FIN, of course) is checked separately. 3704 * See tcp_data_queue(), for example. 3705 * 3706 * Also, controls (RST is main one) are accepted using RCV.WUP instead 3707 * of RCV.NXT. Peer still did not advance his SND.UNA when we 3708 * delayed ACK, so that hisSND.UNA<=ourRCV.WUP. 3709 * (borrowed from freebsd) 3710 */ 3711 3712 static inline bool tcp_sequence(const struct tcp_sock *tp, u32 seq, u32 end_seq) 3713 { 3714 return !before(end_seq, tp->rcv_wup) && 3715 !after(seq, tp->rcv_nxt + tcp_receive_window(tp)); 3716 } 3717 3718 /* When we get a reset we do this. */ 3719 void tcp_reset(struct sock *sk) 3720 { 3721 /* We want the right error as BSD sees it (and indeed as we do). */ 3722 switch (sk->sk_state) { 3723 case TCP_SYN_SENT: 3724 sk->sk_err = ECONNREFUSED; 3725 break; 3726 case TCP_CLOSE_WAIT: 3727 sk->sk_err = EPIPE; 3728 break; 3729 case TCP_CLOSE: 3730 return; 3731 default: 3732 sk->sk_err = ECONNRESET; 3733 } 3734 /* This barrier is coupled with smp_rmb() in tcp_poll() */ 3735 smp_wmb(); 3736 3737 if (!sock_flag(sk, SOCK_DEAD)) 3738 sk->sk_error_report(sk); 3739 3740 tcp_done(sk); 3741 } 3742 3743 /* 3744 * Process the FIN bit. This now behaves as it is supposed to work 3745 * and the FIN takes effect when it is validly part of sequence 3746 * space. Not before when we get holes. 3747 * 3748 * If we are ESTABLISHED, a received fin moves us to CLOSE-WAIT 3749 * (and thence onto LAST-ACK and finally, CLOSE, we never enter 3750 * TIME-WAIT) 3751 * 3752 * If we are in FINWAIT-1, a received FIN indicates simultaneous 3753 * close and we go into CLOSING (and later onto TIME-WAIT) 3754 * 3755 * If we are in FINWAIT-2, a received FIN moves us to TIME-WAIT. 3756 */ 3757 static void tcp_fin(struct sock *sk) 3758 { 3759 struct tcp_sock *tp = tcp_sk(sk); 3760 const struct dst_entry *dst; 3761 3762 inet_csk_schedule_ack(sk); 3763 3764 sk->sk_shutdown |= RCV_SHUTDOWN; 3765 sock_set_flag(sk, SOCK_DONE); 3766 3767 switch (sk->sk_state) { 3768 case TCP_SYN_RECV: 3769 case TCP_ESTABLISHED: 3770 /* Move to CLOSE_WAIT */ 3771 tcp_set_state(sk, TCP_CLOSE_WAIT); 3772 dst = __sk_dst_get(sk); 3773 if (!dst || !dst_metric(dst, RTAX_QUICKACK)) 3774 inet_csk(sk)->icsk_ack.pingpong = 1; 3775 break; 3776 3777 case TCP_CLOSE_WAIT: 3778 case TCP_CLOSING: 3779 /* Received a retransmission of the FIN, do 3780 * nothing. 3781 */ 3782 break; 3783 case TCP_LAST_ACK: 3784 /* RFC793: Remain in the LAST-ACK state. */ 3785 break; 3786 3787 case TCP_FIN_WAIT1: 3788 /* This case occurs when a simultaneous close 3789 * happens, we must ack the received FIN and 3790 * enter the CLOSING state. 3791 */ 3792 tcp_send_ack(sk); 3793 tcp_set_state(sk, TCP_CLOSING); 3794 break; 3795 case TCP_FIN_WAIT2: 3796 /* Received a FIN -- send ACK and enter TIME_WAIT. */ 3797 tcp_send_ack(sk); 3798 tcp_time_wait(sk, TCP_TIME_WAIT, 0); 3799 break; 3800 default: 3801 /* Only TCP_LISTEN and TCP_CLOSE are left, in these 3802 * cases we should never reach this piece of code. 3803 */ 3804 pr_err("%s: Impossible, sk->sk_state=%d\n", 3805 __func__, sk->sk_state); 3806 break; 3807 } 3808 3809 /* It _is_ possible, that we have something out-of-order _after_ FIN. 3810 * Probably, we should reset in this case. For now drop them. 3811 */ 3812 __skb_queue_purge(&tp->out_of_order_queue); 3813 if (tcp_is_sack(tp)) 3814 tcp_sack_reset(&tp->rx_opt); 3815 sk_mem_reclaim(sk); 3816 3817 if (!sock_flag(sk, SOCK_DEAD)) { 3818 sk->sk_state_change(sk); 3819 3820 /* Do not send POLL_HUP for half duplex close. */ 3821 if (sk->sk_shutdown == SHUTDOWN_MASK || 3822 sk->sk_state == TCP_CLOSE) 3823 sk_wake_async(sk, SOCK_WAKE_WAITD, POLL_HUP); 3824 else 3825 sk_wake_async(sk, SOCK_WAKE_WAITD, POLL_IN); 3826 } 3827 } 3828 3829 static inline bool tcp_sack_extend(struct tcp_sack_block *sp, u32 seq, 3830 u32 end_seq) 3831 { 3832 if (!after(seq, sp->end_seq) && !after(sp->start_seq, end_seq)) { 3833 if (before(seq, sp->start_seq)) 3834 sp->start_seq = seq; 3835 if (after(end_seq, sp->end_seq)) 3836 sp->end_seq = end_seq; 3837 return true; 3838 } 3839 return false; 3840 } 3841 3842 static void tcp_dsack_set(struct sock *sk, u32 seq, u32 end_seq) 3843 { 3844 struct tcp_sock *tp = tcp_sk(sk); 3845 3846 if (tcp_is_sack(tp) && sysctl_tcp_dsack) { 3847 int mib_idx; 3848 3849 if (before(seq, tp->rcv_nxt)) 3850 mib_idx = LINUX_MIB_TCPDSACKOLDSENT; 3851 else 3852 mib_idx = LINUX_MIB_TCPDSACKOFOSENT; 3853 3854 NET_INC_STATS_BH(sock_net(sk), mib_idx); 3855 3856 tp->rx_opt.dsack = 1; 3857 tp->duplicate_sack[0].start_seq = seq; 3858 tp->duplicate_sack[0].end_seq = end_seq; 3859 } 3860 } 3861 3862 static void tcp_dsack_extend(struct sock *sk, u32 seq, u32 end_seq) 3863 { 3864 struct tcp_sock *tp = tcp_sk(sk); 3865 3866 if (!tp->rx_opt.dsack) 3867 tcp_dsack_set(sk, seq, end_seq); 3868 else 3869 tcp_sack_extend(tp->duplicate_sack, seq, end_seq); 3870 } 3871 3872 static void tcp_send_dupack(struct sock *sk, const struct sk_buff *skb) 3873 { 3874 struct tcp_sock *tp = tcp_sk(sk); 3875 3876 if (TCP_SKB_CB(skb)->end_seq != TCP_SKB_CB(skb)->seq && 3877 before(TCP_SKB_CB(skb)->seq, tp->rcv_nxt)) { 3878 NET_INC_STATS_BH(sock_net(sk), LINUX_MIB_DELAYEDACKLOST); 3879 tcp_enter_quickack_mode(sk); 3880 3881 if (tcp_is_sack(tp) && sysctl_tcp_dsack) { 3882 u32 end_seq = TCP_SKB_CB(skb)->end_seq; 3883 3884 if (after(TCP_SKB_CB(skb)->end_seq, tp->rcv_nxt)) 3885 end_seq = tp->rcv_nxt; 3886 tcp_dsack_set(sk, TCP_SKB_CB(skb)->seq, end_seq); 3887 } 3888 } 3889 3890 tcp_send_ack(sk); 3891 } 3892 3893 /* These routines update the SACK block as out-of-order packets arrive or 3894 * in-order packets close up the sequence space. 3895 */ 3896 static void tcp_sack_maybe_coalesce(struct tcp_sock *tp) 3897 { 3898 int this_sack; 3899 struct tcp_sack_block *sp = &tp->selective_acks[0]; 3900 struct tcp_sack_block *swalk = sp + 1; 3901 3902 /* See if the recent change to the first SACK eats into 3903 * or hits the sequence space of other SACK blocks, if so coalesce. 3904 */ 3905 for (this_sack = 1; this_sack < tp->rx_opt.num_sacks;) { 3906 if (tcp_sack_extend(sp, swalk->start_seq, swalk->end_seq)) { 3907 int i; 3908 3909 /* Zap SWALK, by moving every further SACK up by one slot. 3910 * Decrease num_sacks. 3911 */ 3912 tp->rx_opt.num_sacks--; 3913 for (i = this_sack; i < tp->rx_opt.num_sacks; i++) 3914 sp[i] = sp[i + 1]; 3915 continue; 3916 } 3917 this_sack++, swalk++; 3918 } 3919 } 3920 3921 static void tcp_sack_new_ofo_skb(struct sock *sk, u32 seq, u32 end_seq) 3922 { 3923 struct tcp_sock *tp = tcp_sk(sk); 3924 struct tcp_sack_block *sp = &tp->selective_acks[0]; 3925 int cur_sacks = tp->rx_opt.num_sacks; 3926 int this_sack; 3927 3928 if (!cur_sacks) 3929 goto new_sack; 3930 3931 for (this_sack = 0; this_sack < cur_sacks; this_sack++, sp++) { 3932 if (tcp_sack_extend(sp, seq, end_seq)) { 3933 /* Rotate this_sack to the first one. */ 3934 for (; this_sack > 0; this_sack--, sp--) 3935 swap(*sp, *(sp - 1)); 3936 if (cur_sacks > 1) 3937 tcp_sack_maybe_coalesce(tp); 3938 return; 3939 } 3940 } 3941 3942 /* Could not find an adjacent existing SACK, build a new one, 3943 * put it at the front, and shift everyone else down. We 3944 * always know there is at least one SACK present already here. 3945 * 3946 * If the sack array is full, forget about the last one. 3947 */ 3948 if (this_sack >= TCP_NUM_SACKS) { 3949 this_sack--; 3950 tp->rx_opt.num_sacks--; 3951 sp--; 3952 } 3953 for (; this_sack > 0; this_sack--, sp--) 3954 *sp = *(sp - 1); 3955 3956 new_sack: 3957 /* Build the new head SACK, and we're done. */ 3958 sp->start_seq = seq; 3959 sp->end_seq = end_seq; 3960 tp->rx_opt.num_sacks++; 3961 } 3962 3963 /* RCV.NXT advances, some SACKs should be eaten. */ 3964 3965 static void tcp_sack_remove(struct tcp_sock *tp) 3966 { 3967 struct tcp_sack_block *sp = &tp->selective_acks[0]; 3968 int num_sacks = tp->rx_opt.num_sacks; 3969 int this_sack; 3970 3971 /* Empty ofo queue, hence, all the SACKs are eaten. Clear. */ 3972 if (skb_queue_empty(&tp->out_of_order_queue)) { 3973 tp->rx_opt.num_sacks = 0; 3974 return; 3975 } 3976 3977 for (this_sack = 0; this_sack < num_sacks;) { 3978 /* Check if the start of the sack is covered by RCV.NXT. */ 3979 if (!before(tp->rcv_nxt, sp->start_seq)) { 3980 int i; 3981 3982 /* RCV.NXT must cover all the block! */ 3983 WARN_ON(before(tp->rcv_nxt, sp->end_seq)); 3984 3985 /* Zap this SACK, by moving forward any other SACKS. */ 3986 for (i=this_sack+1; i < num_sacks; i++) 3987 tp->selective_acks[i-1] = tp->selective_acks[i]; 3988 num_sacks--; 3989 continue; 3990 } 3991 this_sack++; 3992 sp++; 3993 } 3994 tp->rx_opt.num_sacks = num_sacks; 3995 } 3996 3997 /* This one checks to see if we can put data from the 3998 * out_of_order queue into the receive_queue. 3999 */ 4000 static void tcp_ofo_queue(struct sock *sk) 4001 { 4002 struct tcp_sock *tp = tcp_sk(sk); 4003 __u32 dsack_high = tp->rcv_nxt; 4004 struct sk_buff *skb; 4005 4006 while ((skb = skb_peek(&tp->out_of_order_queue)) != NULL) { 4007 if (after(TCP_SKB_CB(skb)->seq, tp->rcv_nxt)) 4008 break; 4009 4010 if (before(TCP_SKB_CB(skb)->seq, dsack_high)) { 4011 __u32 dsack = dsack_high; 4012 if (before(TCP_SKB_CB(skb)->end_seq, dsack_high)) 4013 dsack_high = TCP_SKB_CB(skb)->end_seq; 4014 tcp_dsack_extend(sk, TCP_SKB_CB(skb)->seq, dsack); 4015 } 4016 4017 if (!after(TCP_SKB_CB(skb)->end_seq, tp->rcv_nxt)) { 4018 SOCK_DEBUG(sk, "ofo packet was already received\n"); 4019 __skb_unlink(skb, &tp->out_of_order_queue); 4020 __kfree_skb(skb); 4021 continue; 4022 } 4023 SOCK_DEBUG(sk, "ofo requeuing : rcv_next %X seq %X - %X\n", 4024 tp->rcv_nxt, TCP_SKB_CB(skb)->seq, 4025 TCP_SKB_CB(skb)->end_seq); 4026 4027 __skb_unlink(skb, &tp->out_of_order_queue); 4028 __skb_queue_tail(&sk->sk_receive_queue, skb); 4029 tp->rcv_nxt = TCP_SKB_CB(skb)->end_seq; 4030 if (tcp_hdr(skb)->fin) 4031 tcp_fin(sk); 4032 } 4033 } 4034 4035 static bool tcp_prune_ofo_queue(struct sock *sk); 4036 static int tcp_prune_queue(struct sock *sk); 4037 4038 static int tcp_try_rmem_schedule(struct sock *sk, struct sk_buff *skb, 4039 unsigned int size) 4040 { 4041 if (atomic_read(&sk->sk_rmem_alloc) > sk->sk_rcvbuf || 4042 !sk_rmem_schedule(sk, skb, size)) { 4043 4044 if (tcp_prune_queue(sk) < 0) 4045 return -1; 4046 4047 if (!sk_rmem_schedule(sk, skb, size)) { 4048 if (!tcp_prune_ofo_queue(sk)) 4049 return -1; 4050 4051 if (!sk_rmem_schedule(sk, skb, size)) 4052 return -1; 4053 } 4054 } 4055 return 0; 4056 } 4057 4058 /** 4059 * tcp_try_coalesce - try to merge skb to prior one 4060 * @sk: socket 4061 * @to: prior buffer 4062 * @from: buffer to add in queue 4063 * @fragstolen: pointer to boolean 4064 * 4065 * Before queueing skb @from after @to, try to merge them 4066 * to reduce overall memory use and queue lengths, if cost is small. 4067 * Packets in ofo or receive queues can stay a long time. 4068 * Better try to coalesce them right now to avoid future collapses. 4069 * Returns true if caller should free @from instead of queueing it 4070 */ 4071 static bool tcp_try_coalesce(struct sock *sk, 4072 struct sk_buff *to, 4073 struct sk_buff *from, 4074 bool *fragstolen) 4075 { 4076 int delta; 4077 4078 *fragstolen = false; 4079 4080 if (tcp_hdr(from)->fin) 4081 return false; 4082 4083 /* Its possible this segment overlaps with prior segment in queue */ 4084 if (TCP_SKB_CB(from)->seq != TCP_SKB_CB(to)->end_seq) 4085 return false; 4086 4087 if (!skb_try_coalesce(to, from, fragstolen, &delta)) 4088 return false; 4089 4090 atomic_add(delta, &sk->sk_rmem_alloc); 4091 sk_mem_charge(sk, delta); 4092 NET_INC_STATS_BH(sock_net(sk), LINUX_MIB_TCPRCVCOALESCE); 4093 TCP_SKB_CB(to)->end_seq = TCP_SKB_CB(from)->end_seq; 4094 TCP_SKB_CB(to)->ack_seq = TCP_SKB_CB(from)->ack_seq; 4095 return true; 4096 } 4097 4098 static void tcp_data_queue_ofo(struct sock *sk, struct sk_buff *skb) 4099 { 4100 struct tcp_sock *tp = tcp_sk(sk); 4101 struct sk_buff *skb1; 4102 u32 seq, end_seq; 4103 4104 TCP_ECN_check_ce(tp, skb); 4105 4106 if (unlikely(tcp_try_rmem_schedule(sk, skb, skb->truesize))) { 4107 NET_INC_STATS_BH(sock_net(sk), LINUX_MIB_TCPOFODROP); 4108 __kfree_skb(skb); 4109 return; 4110 } 4111 4112 /* Disable header prediction. */ 4113 tp->pred_flags = 0; 4114 inet_csk_schedule_ack(sk); 4115 4116 NET_INC_STATS_BH(sock_net(sk), LINUX_MIB_TCPOFOQUEUE); 4117 SOCK_DEBUG(sk, "out of order segment: rcv_next %X seq %X - %X\n", 4118 tp->rcv_nxt, TCP_SKB_CB(skb)->seq, TCP_SKB_CB(skb)->end_seq); 4119 4120 skb1 = skb_peek_tail(&tp->out_of_order_queue); 4121 if (!skb1) { 4122 /* Initial out of order segment, build 1 SACK. */ 4123 if (tcp_is_sack(tp)) { 4124 tp->rx_opt.num_sacks = 1; 4125 tp->selective_acks[0].start_seq = TCP_SKB_CB(skb)->seq; 4126 tp->selective_acks[0].end_seq = 4127 TCP_SKB_CB(skb)->end_seq; 4128 } 4129 __skb_queue_head(&tp->out_of_order_queue, skb); 4130 goto end; 4131 } 4132 4133 seq = TCP_SKB_CB(skb)->seq; 4134 end_seq = TCP_SKB_CB(skb)->end_seq; 4135 4136 if (seq == TCP_SKB_CB(skb1)->end_seq) { 4137 bool fragstolen; 4138 4139 if (!tcp_try_coalesce(sk, skb1, skb, &fragstolen)) { 4140 __skb_queue_after(&tp->out_of_order_queue, skb1, skb); 4141 } else { 4142 tcp_grow_window(sk, skb); 4143 kfree_skb_partial(skb, fragstolen); 4144 skb = NULL; 4145 } 4146 4147 if (!tp->rx_opt.num_sacks || 4148 tp->selective_acks[0].end_seq != seq) 4149 goto add_sack; 4150 4151 /* Common case: data arrive in order after hole. */ 4152 tp->selective_acks[0].end_seq = end_seq; 4153 goto end; 4154 } 4155 4156 /* Find place to insert this segment. */ 4157 while (1) { 4158 if (!after(TCP_SKB_CB(skb1)->seq, seq)) 4159 break; 4160 if (skb_queue_is_first(&tp->out_of_order_queue, skb1)) { 4161 skb1 = NULL; 4162 break; 4163 } 4164 skb1 = skb_queue_prev(&tp->out_of_order_queue, skb1); 4165 } 4166 4167 /* Do skb overlap to previous one? */ 4168 if (skb1 && before(seq, TCP_SKB_CB(skb1)->end_seq)) { 4169 if (!after(end_seq, TCP_SKB_CB(skb1)->end_seq)) { 4170 /* All the bits are present. Drop. */ 4171 NET_INC_STATS_BH(sock_net(sk), LINUX_MIB_TCPOFOMERGE); 4172 __kfree_skb(skb); 4173 skb = NULL; 4174 tcp_dsack_set(sk, seq, end_seq); 4175 goto add_sack; 4176 } 4177 if (after(seq, TCP_SKB_CB(skb1)->seq)) { 4178 /* Partial overlap. */ 4179 tcp_dsack_set(sk, seq, 4180 TCP_SKB_CB(skb1)->end_seq); 4181 } else { 4182 if (skb_queue_is_first(&tp->out_of_order_queue, 4183 skb1)) 4184 skb1 = NULL; 4185 else 4186 skb1 = skb_queue_prev( 4187 &tp->out_of_order_queue, 4188 skb1); 4189 } 4190 } 4191 if (!skb1) 4192 __skb_queue_head(&tp->out_of_order_queue, skb); 4193 else 4194 __skb_queue_after(&tp->out_of_order_queue, skb1, skb); 4195 4196 /* And clean segments covered by new one as whole. */ 4197 while (!skb_queue_is_last(&tp->out_of_order_queue, skb)) { 4198 skb1 = skb_queue_next(&tp->out_of_order_queue, skb); 4199 4200 if (!after(end_seq, TCP_SKB_CB(skb1)->seq)) 4201 break; 4202 if (before(end_seq, TCP_SKB_CB(skb1)->end_seq)) { 4203 tcp_dsack_extend(sk, TCP_SKB_CB(skb1)->seq, 4204 end_seq); 4205 break; 4206 } 4207 __skb_unlink(skb1, &tp->out_of_order_queue); 4208 tcp_dsack_extend(sk, TCP_SKB_CB(skb1)->seq, 4209 TCP_SKB_CB(skb1)->end_seq); 4210 NET_INC_STATS_BH(sock_net(sk), LINUX_MIB_TCPOFOMERGE); 4211 __kfree_skb(skb1); 4212 } 4213 4214 add_sack: 4215 if (tcp_is_sack(tp)) 4216 tcp_sack_new_ofo_skb(sk, seq, end_seq); 4217 end: 4218 if (skb) { 4219 tcp_grow_window(sk, skb); 4220 skb_set_owner_r(skb, sk); 4221 } 4222 } 4223 4224 static int __must_check tcp_queue_rcv(struct sock *sk, struct sk_buff *skb, int hdrlen, 4225 bool *fragstolen) 4226 { 4227 int eaten; 4228 struct sk_buff *tail = skb_peek_tail(&sk->sk_receive_queue); 4229 4230 __skb_pull(skb, hdrlen); 4231 eaten = (tail && 4232 tcp_try_coalesce(sk, tail, skb, fragstolen)) ? 1 : 0; 4233 tcp_sk(sk)->rcv_nxt = TCP_SKB_CB(skb)->end_seq; 4234 if (!eaten) { 4235 __skb_queue_tail(&sk->sk_receive_queue, skb); 4236 skb_set_owner_r(skb, sk); 4237 } 4238 return eaten; 4239 } 4240 4241 int tcp_send_rcvq(struct sock *sk, struct msghdr *msg, size_t size) 4242 { 4243 struct sk_buff *skb = NULL; 4244 struct tcphdr *th; 4245 bool fragstolen; 4246 4247 if (size == 0) 4248 return 0; 4249 4250 skb = alloc_skb(size + sizeof(*th), sk->sk_allocation); 4251 if (!skb) 4252 goto err; 4253 4254 if (tcp_try_rmem_schedule(sk, skb, size + sizeof(*th))) 4255 goto err_free; 4256 4257 th = (struct tcphdr *)skb_put(skb, sizeof(*th)); 4258 skb_reset_transport_header(skb); 4259 memset(th, 0, sizeof(*th)); 4260 4261 if (memcpy_fromiovec(skb_put(skb, size), msg->msg_iov, size)) 4262 goto err_free; 4263 4264 TCP_SKB_CB(skb)->seq = tcp_sk(sk)->rcv_nxt; 4265 TCP_SKB_CB(skb)->end_seq = TCP_SKB_CB(skb)->seq + size; 4266 TCP_SKB_CB(skb)->ack_seq = tcp_sk(sk)->snd_una - 1; 4267 4268 if (tcp_queue_rcv(sk, skb, sizeof(*th), &fragstolen)) { 4269 WARN_ON_ONCE(fragstolen); /* should not happen */ 4270 __kfree_skb(skb); 4271 } 4272 return size; 4273 4274 err_free: 4275 kfree_skb(skb); 4276 err: 4277 return -ENOMEM; 4278 } 4279 4280 static void tcp_data_queue(struct sock *sk, struct sk_buff *skb) 4281 { 4282 const struct tcphdr *th = tcp_hdr(skb); 4283 struct tcp_sock *tp = tcp_sk(sk); 4284 int eaten = -1; 4285 bool fragstolen = false; 4286 4287 if (TCP_SKB_CB(skb)->seq == TCP_SKB_CB(skb)->end_seq) 4288 goto drop; 4289 4290 skb_dst_drop(skb); 4291 __skb_pull(skb, th->doff * 4); 4292 4293 TCP_ECN_accept_cwr(tp, skb); 4294 4295 tp->rx_opt.dsack = 0; 4296 4297 /* Queue data for delivery to the user. 4298 * Packets in sequence go to the receive queue. 4299 * Out of sequence packets to the out_of_order_queue. 4300 */ 4301 if (TCP_SKB_CB(skb)->seq == tp->rcv_nxt) { 4302 if (tcp_receive_window(tp) == 0) 4303 goto out_of_window; 4304 4305 /* Ok. In sequence. In window. */ 4306 if (tp->ucopy.task == current && 4307 tp->copied_seq == tp->rcv_nxt && tp->ucopy.len && 4308 sock_owned_by_user(sk) && !tp->urg_data) { 4309 int chunk = min_t(unsigned int, skb->len, 4310 tp->ucopy.len); 4311 4312 __set_current_state(TASK_RUNNING); 4313 4314 local_bh_enable(); 4315 if (!skb_copy_datagram_iovec(skb, 0, tp->ucopy.iov, chunk)) { 4316 tp->ucopy.len -= chunk; 4317 tp->copied_seq += chunk; 4318 eaten = (chunk == skb->len); 4319 tcp_rcv_space_adjust(sk); 4320 } 4321 local_bh_disable(); 4322 } 4323 4324 if (eaten <= 0) { 4325 queue_and_out: 4326 if (eaten < 0 && 4327 tcp_try_rmem_schedule(sk, skb, skb->truesize)) 4328 goto drop; 4329 4330 eaten = tcp_queue_rcv(sk, skb, 0, &fragstolen); 4331 } 4332 tp->rcv_nxt = TCP_SKB_CB(skb)->end_seq; 4333 if (skb->len) 4334 tcp_event_data_recv(sk, skb); 4335 if (th->fin) 4336 tcp_fin(sk); 4337 4338 if (!skb_queue_empty(&tp->out_of_order_queue)) { 4339 tcp_ofo_queue(sk); 4340 4341 /* RFC2581. 4.2. SHOULD send immediate ACK, when 4342 * gap in queue is filled. 4343 */ 4344 if (skb_queue_empty(&tp->out_of_order_queue)) 4345 inet_csk(sk)->icsk_ack.pingpong = 0; 4346 } 4347 4348 if (tp->rx_opt.num_sacks) 4349 tcp_sack_remove(tp); 4350 4351 tcp_fast_path_check(sk); 4352 4353 if (eaten > 0) 4354 kfree_skb_partial(skb, fragstolen); 4355 if (!sock_flag(sk, SOCK_DEAD)) 4356 sk->sk_data_ready(sk, 0); 4357 return; 4358 } 4359 4360 if (!after(TCP_SKB_CB(skb)->end_seq, tp->rcv_nxt)) { 4361 /* A retransmit, 2nd most common case. Force an immediate ack. */ 4362 NET_INC_STATS_BH(sock_net(sk), LINUX_MIB_DELAYEDACKLOST); 4363 tcp_dsack_set(sk, TCP_SKB_CB(skb)->seq, TCP_SKB_CB(skb)->end_seq); 4364 4365 out_of_window: 4366 tcp_enter_quickack_mode(sk); 4367 inet_csk_schedule_ack(sk); 4368 drop: 4369 __kfree_skb(skb); 4370 return; 4371 } 4372 4373 /* Out of window. F.e. zero window probe. */ 4374 if (!before(TCP_SKB_CB(skb)->seq, tp->rcv_nxt + tcp_receive_window(tp))) 4375 goto out_of_window; 4376 4377 tcp_enter_quickack_mode(sk); 4378 4379 if (before(TCP_SKB_CB(skb)->seq, tp->rcv_nxt)) { 4380 /* Partial packet, seq < rcv_next < end_seq */ 4381 SOCK_DEBUG(sk, "partial packet: rcv_next %X seq %X - %X\n", 4382 tp->rcv_nxt, TCP_SKB_CB(skb)->seq, 4383 TCP_SKB_CB(skb)->end_seq); 4384 4385 tcp_dsack_set(sk, TCP_SKB_CB(skb)->seq, tp->rcv_nxt); 4386 4387 /* If window is closed, drop tail of packet. But after 4388 * remembering D-SACK for its head made in previous line. 4389 */ 4390 if (!tcp_receive_window(tp)) 4391 goto out_of_window; 4392 goto queue_and_out; 4393 } 4394 4395 tcp_data_queue_ofo(sk, skb); 4396 } 4397 4398 static struct sk_buff *tcp_collapse_one(struct sock *sk, struct sk_buff *skb, 4399 struct sk_buff_head *list) 4400 { 4401 struct sk_buff *next = NULL; 4402 4403 if (!skb_queue_is_last(list, skb)) 4404 next = skb_queue_next(list, skb); 4405 4406 __skb_unlink(skb, list); 4407 __kfree_skb(skb); 4408 NET_INC_STATS_BH(sock_net(sk), LINUX_MIB_TCPRCVCOLLAPSED); 4409 4410 return next; 4411 } 4412 4413 /* Collapse contiguous sequence of skbs head..tail with 4414 * sequence numbers start..end. 4415 * 4416 * If tail is NULL, this means until the end of the list. 4417 * 4418 * Segments with FIN/SYN are not collapsed (only because this 4419 * simplifies code) 4420 */ 4421 static void 4422 tcp_collapse(struct sock *sk, struct sk_buff_head *list, 4423 struct sk_buff *head, struct sk_buff *tail, 4424 u32 start, u32 end) 4425 { 4426 struct sk_buff *skb, *n; 4427 bool end_of_skbs; 4428 4429 /* First, check that queue is collapsible and find 4430 * the point where collapsing can be useful. */ 4431 skb = head; 4432 restart: 4433 end_of_skbs = true; 4434 skb_queue_walk_from_safe(list, skb, n) { 4435 if (skb == tail) 4436 break; 4437 /* No new bits? It is possible on ofo queue. */ 4438 if (!before(start, TCP_SKB_CB(skb)->end_seq)) { 4439 skb = tcp_collapse_one(sk, skb, list); 4440 if (!skb) 4441 break; 4442 goto restart; 4443 } 4444 4445 /* The first skb to collapse is: 4446 * - not SYN/FIN and 4447 * - bloated or contains data before "start" or 4448 * overlaps to the next one. 4449 */ 4450 if (!tcp_hdr(skb)->syn && !tcp_hdr(skb)->fin && 4451 (tcp_win_from_space(skb->truesize) > skb->len || 4452 before(TCP_SKB_CB(skb)->seq, start))) { 4453 end_of_skbs = false; 4454 break; 4455 } 4456 4457 if (!skb_queue_is_last(list, skb)) { 4458 struct sk_buff *next = skb_queue_next(list, skb); 4459 if (next != tail && 4460 TCP_SKB_CB(skb)->end_seq != TCP_SKB_CB(next)->seq) { 4461 end_of_skbs = false; 4462 break; 4463 } 4464 } 4465 4466 /* Decided to skip this, advance start seq. */ 4467 start = TCP_SKB_CB(skb)->end_seq; 4468 } 4469 if (end_of_skbs || tcp_hdr(skb)->syn || tcp_hdr(skb)->fin) 4470 return; 4471 4472 while (before(start, end)) { 4473 struct sk_buff *nskb; 4474 unsigned int header = skb_headroom(skb); 4475 int copy = SKB_MAX_ORDER(header, 0); 4476 4477 /* Too big header? This can happen with IPv6. */ 4478 if (copy < 0) 4479 return; 4480 if (end - start < copy) 4481 copy = end - start; 4482 nskb = alloc_skb(copy + header, GFP_ATOMIC); 4483 if (!nskb) 4484 return; 4485 4486 skb_set_mac_header(nskb, skb_mac_header(skb) - skb->head); 4487 skb_set_network_header(nskb, (skb_network_header(skb) - 4488 skb->head)); 4489 skb_set_transport_header(nskb, (skb_transport_header(skb) - 4490 skb->head)); 4491 skb_reserve(nskb, header); 4492 memcpy(nskb->head, skb->head, header); 4493 memcpy(nskb->cb, skb->cb, sizeof(skb->cb)); 4494 TCP_SKB_CB(nskb)->seq = TCP_SKB_CB(nskb)->end_seq = start; 4495 __skb_queue_before(list, skb, nskb); 4496 skb_set_owner_r(nskb, sk); 4497 4498 /* Copy data, releasing collapsed skbs. */ 4499 while (copy > 0) { 4500 int offset = start - TCP_SKB_CB(skb)->seq; 4501 int size = TCP_SKB_CB(skb)->end_seq - start; 4502 4503 BUG_ON(offset < 0); 4504 if (size > 0) { 4505 size = min(copy, size); 4506 if (skb_copy_bits(skb, offset, skb_put(nskb, size), size)) 4507 BUG(); 4508 TCP_SKB_CB(nskb)->end_seq += size; 4509 copy -= size; 4510 start += size; 4511 } 4512 if (!before(start, TCP_SKB_CB(skb)->end_seq)) { 4513 skb = tcp_collapse_one(sk, skb, list); 4514 if (!skb || 4515 skb == tail || 4516 tcp_hdr(skb)->syn || 4517 tcp_hdr(skb)->fin) 4518 return; 4519 } 4520 } 4521 } 4522 } 4523 4524 /* Collapse ofo queue. Algorithm: select contiguous sequence of skbs 4525 * and tcp_collapse() them until all the queue is collapsed. 4526 */ 4527 static void tcp_collapse_ofo_queue(struct sock *sk) 4528 { 4529 struct tcp_sock *tp = tcp_sk(sk); 4530 struct sk_buff *skb = skb_peek(&tp->out_of_order_queue); 4531 struct sk_buff *head; 4532 u32 start, end; 4533 4534 if (skb == NULL) 4535 return; 4536 4537 start = TCP_SKB_CB(skb)->seq; 4538 end = TCP_SKB_CB(skb)->end_seq; 4539 head = skb; 4540 4541 for (;;) { 4542 struct sk_buff *next = NULL; 4543 4544 if (!skb_queue_is_last(&tp->out_of_order_queue, skb)) 4545 next = skb_queue_next(&tp->out_of_order_queue, skb); 4546 skb = next; 4547 4548 /* Segment is terminated when we see gap or when 4549 * we are at the end of all the queue. */ 4550 if (!skb || 4551 after(TCP_SKB_CB(skb)->seq, end) || 4552 before(TCP_SKB_CB(skb)->end_seq, start)) { 4553 tcp_collapse(sk, &tp->out_of_order_queue, 4554 head, skb, start, end); 4555 head = skb; 4556 if (!skb) 4557 break; 4558 /* Start new segment */ 4559 start = TCP_SKB_CB(skb)->seq; 4560 end = TCP_SKB_CB(skb)->end_seq; 4561 } else { 4562 if (before(TCP_SKB_CB(skb)->seq, start)) 4563 start = TCP_SKB_CB(skb)->seq; 4564 if (after(TCP_SKB_CB(skb)->end_seq, end)) 4565 end = TCP_SKB_CB(skb)->end_seq; 4566 } 4567 } 4568 } 4569 4570 /* 4571 * Purge the out-of-order queue. 4572 * Return true if queue was pruned. 4573 */ 4574 static bool tcp_prune_ofo_queue(struct sock *sk) 4575 { 4576 struct tcp_sock *tp = tcp_sk(sk); 4577 bool res = false; 4578 4579 if (!skb_queue_empty(&tp->out_of_order_queue)) { 4580 NET_INC_STATS_BH(sock_net(sk), LINUX_MIB_OFOPRUNED); 4581 __skb_queue_purge(&tp->out_of_order_queue); 4582 4583 /* Reset SACK state. A conforming SACK implementation will 4584 * do the same at a timeout based retransmit. When a connection 4585 * is in a sad state like this, we care only about integrity 4586 * of the connection not performance. 4587 */ 4588 if (tp->rx_opt.sack_ok) 4589 tcp_sack_reset(&tp->rx_opt); 4590 sk_mem_reclaim(sk); 4591 res = true; 4592 } 4593 return res; 4594 } 4595 4596 /* Reduce allocated memory if we can, trying to get 4597 * the socket within its memory limits again. 4598 * 4599 * Return less than zero if we should start dropping frames 4600 * until the socket owning process reads some of the data 4601 * to stabilize the situation. 4602 */ 4603 static int tcp_prune_queue(struct sock *sk) 4604 { 4605 struct tcp_sock *tp = tcp_sk(sk); 4606 4607 SOCK_DEBUG(sk, "prune_queue: c=%x\n", tp->copied_seq); 4608 4609 NET_INC_STATS_BH(sock_net(sk), LINUX_MIB_PRUNECALLED); 4610 4611 if (atomic_read(&sk->sk_rmem_alloc) >= sk->sk_rcvbuf) 4612 tcp_clamp_window(sk); 4613 else if (sk_under_memory_pressure(sk)) 4614 tp->rcv_ssthresh = min(tp->rcv_ssthresh, 4U * tp->advmss); 4615 4616 tcp_collapse_ofo_queue(sk); 4617 if (!skb_queue_empty(&sk->sk_receive_queue)) 4618 tcp_collapse(sk, &sk->sk_receive_queue, 4619 skb_peek(&sk->sk_receive_queue), 4620 NULL, 4621 tp->copied_seq, tp->rcv_nxt); 4622 sk_mem_reclaim(sk); 4623 4624 if (atomic_read(&sk->sk_rmem_alloc) <= sk->sk_rcvbuf) 4625 return 0; 4626 4627 /* Collapsing did not help, destructive actions follow. 4628 * This must not ever occur. */ 4629 4630 tcp_prune_ofo_queue(sk); 4631 4632 if (atomic_read(&sk->sk_rmem_alloc) <= sk->sk_rcvbuf) 4633 return 0; 4634 4635 /* If we are really being abused, tell the caller to silently 4636 * drop receive data on the floor. It will get retransmitted 4637 * and hopefully then we'll have sufficient space. 4638 */ 4639 NET_INC_STATS_BH(sock_net(sk), LINUX_MIB_RCVPRUNED); 4640 4641 /* Massive buffer overcommit. */ 4642 tp->pred_flags = 0; 4643 return -1; 4644 } 4645 4646 /* RFC2861, slow part. Adjust cwnd, after it was not full during one rto. 4647 * As additional protections, we do not touch cwnd in retransmission phases, 4648 * and if application hit its sndbuf limit recently. 4649 */ 4650 void tcp_cwnd_application_limited(struct sock *sk) 4651 { 4652 struct tcp_sock *tp = tcp_sk(sk); 4653 4654 if (inet_csk(sk)->icsk_ca_state == TCP_CA_Open && 4655 sk->sk_socket && !test_bit(SOCK_NOSPACE, &sk->sk_socket->flags)) { 4656 /* Limited by application or receiver window. */ 4657 u32 init_win = tcp_init_cwnd(tp, __sk_dst_get(sk)); 4658 u32 win_used = max(tp->snd_cwnd_used, init_win); 4659 if (win_used < tp->snd_cwnd) { 4660 tp->snd_ssthresh = tcp_current_ssthresh(sk); 4661 tp->snd_cwnd = (tp->snd_cwnd + win_used) >> 1; 4662 } 4663 tp->snd_cwnd_used = 0; 4664 } 4665 tp->snd_cwnd_stamp = tcp_time_stamp; 4666 } 4667 4668 static bool tcp_should_expand_sndbuf(const struct sock *sk) 4669 { 4670 const struct tcp_sock *tp = tcp_sk(sk); 4671 4672 /* If the user specified a specific send buffer setting, do 4673 * not modify it. 4674 */ 4675 if (sk->sk_userlocks & SOCK_SNDBUF_LOCK) 4676 return false; 4677 4678 /* If we are under global TCP memory pressure, do not expand. */ 4679 if (sk_under_memory_pressure(sk)) 4680 return false; 4681 4682 /* If we are under soft global TCP memory pressure, do not expand. */ 4683 if (sk_memory_allocated(sk) >= sk_prot_mem_limits(sk, 0)) 4684 return false; 4685 4686 /* If we filled the congestion window, do not expand. */ 4687 if (tp->packets_out >= tp->snd_cwnd) 4688 return false; 4689 4690 return true; 4691 } 4692 4693 /* When incoming ACK allowed to free some skb from write_queue, 4694 * we remember this event in flag SOCK_QUEUE_SHRUNK and wake up socket 4695 * on the exit from tcp input handler. 4696 * 4697 * PROBLEM: sndbuf expansion does not work well with largesend. 4698 */ 4699 static void tcp_new_space(struct sock *sk) 4700 { 4701 struct tcp_sock *tp = tcp_sk(sk); 4702 4703 if (tcp_should_expand_sndbuf(sk)) { 4704 int sndmem = SKB_TRUESIZE(max_t(u32, 4705 tp->rx_opt.mss_clamp, 4706 tp->mss_cache) + 4707 MAX_TCP_HEADER); 4708 int demanded = max_t(unsigned int, tp->snd_cwnd, 4709 tp->reordering + 1); 4710 sndmem *= 2 * demanded; 4711 if (sndmem > sk->sk_sndbuf) 4712 sk->sk_sndbuf = min(sndmem, sysctl_tcp_wmem[2]); 4713 tp->snd_cwnd_stamp = tcp_time_stamp; 4714 } 4715 4716 sk->sk_write_space(sk); 4717 } 4718 4719 static void tcp_check_space(struct sock *sk) 4720 { 4721 if (sock_flag(sk, SOCK_QUEUE_SHRUNK)) { 4722 sock_reset_flag(sk, SOCK_QUEUE_SHRUNK); 4723 if (sk->sk_socket && 4724 test_bit(SOCK_NOSPACE, &sk->sk_socket->flags)) 4725 tcp_new_space(sk); 4726 } 4727 } 4728 4729 static inline void tcp_data_snd_check(struct sock *sk) 4730 { 4731 tcp_push_pending_frames(sk); 4732 tcp_check_space(sk); 4733 } 4734 4735 /* 4736 * Check if sending an ack is needed. 4737 */ 4738 static void __tcp_ack_snd_check(struct sock *sk, int ofo_possible) 4739 { 4740 struct tcp_sock *tp = tcp_sk(sk); 4741 4742 /* More than one full frame received... */ 4743 if (((tp->rcv_nxt - tp->rcv_wup) > inet_csk(sk)->icsk_ack.rcv_mss && 4744 /* ... and right edge of window advances far enough. 4745 * (tcp_recvmsg() will send ACK otherwise). Or... 4746 */ 4747 __tcp_select_window(sk) >= tp->rcv_wnd) || 4748 /* We ACK each frame or... */ 4749 tcp_in_quickack_mode(sk) || 4750 /* We have out of order data. */ 4751 (ofo_possible && skb_peek(&tp->out_of_order_queue))) { 4752 /* Then ack it now */ 4753 tcp_send_ack(sk); 4754 } else { 4755 /* Else, send delayed ack. */ 4756 tcp_send_delayed_ack(sk); 4757 } 4758 } 4759 4760 static inline void tcp_ack_snd_check(struct sock *sk) 4761 { 4762 if (!inet_csk_ack_scheduled(sk)) { 4763 /* We sent a data segment already. */ 4764 return; 4765 } 4766 __tcp_ack_snd_check(sk, 1); 4767 } 4768 4769 /* 4770 * This routine is only called when we have urgent data 4771 * signaled. Its the 'slow' part of tcp_urg. It could be 4772 * moved inline now as tcp_urg is only called from one 4773 * place. We handle URGent data wrong. We have to - as 4774 * BSD still doesn't use the correction from RFC961. 4775 * For 1003.1g we should support a new option TCP_STDURG to permit 4776 * either form (or just set the sysctl tcp_stdurg). 4777 */ 4778 4779 static void tcp_check_urg(struct sock *sk, const struct tcphdr *th) 4780 { 4781 struct tcp_sock *tp = tcp_sk(sk); 4782 u32 ptr = ntohs(th->urg_ptr); 4783 4784 if (ptr && !sysctl_tcp_stdurg) 4785 ptr--; 4786 ptr += ntohl(th->seq); 4787 4788 /* Ignore urgent data that we've already seen and read. */ 4789 if (after(tp->copied_seq, ptr)) 4790 return; 4791 4792 /* Do not replay urg ptr. 4793 * 4794 * NOTE: interesting situation not covered by specs. 4795 * Misbehaving sender may send urg ptr, pointing to segment, 4796 * which we already have in ofo queue. We are not able to fetch 4797 * such data and will stay in TCP_URG_NOTYET until will be eaten 4798 * by recvmsg(). Seems, we are not obliged to handle such wicked 4799 * situations. But it is worth to think about possibility of some 4800 * DoSes using some hypothetical application level deadlock. 4801 */ 4802 if (before(ptr, tp->rcv_nxt)) 4803 return; 4804 4805 /* Do we already have a newer (or duplicate) urgent pointer? */ 4806 if (tp->urg_data && !after(ptr, tp->urg_seq)) 4807 return; 4808 4809 /* Tell the world about our new urgent pointer. */ 4810 sk_send_sigurg(sk); 4811 4812 /* We may be adding urgent data when the last byte read was 4813 * urgent. To do this requires some care. We cannot just ignore 4814 * tp->copied_seq since we would read the last urgent byte again 4815 * as data, nor can we alter copied_seq until this data arrives 4816 * or we break the semantics of SIOCATMARK (and thus sockatmark()) 4817 * 4818 * NOTE. Double Dutch. Rendering to plain English: author of comment 4819 * above did something sort of send("A", MSG_OOB); send("B", MSG_OOB); 4820 * and expect that both A and B disappear from stream. This is _wrong_. 4821 * Though this happens in BSD with high probability, this is occasional. 4822 * Any application relying on this is buggy. Note also, that fix "works" 4823 * only in this artificial test. Insert some normal data between A and B and we will 4824 * decline of BSD again. Verdict: it is better to remove to trap 4825 * buggy users. 4826 */ 4827 if (tp->urg_seq == tp->copied_seq && tp->urg_data && 4828 !sock_flag(sk, SOCK_URGINLINE) && tp->copied_seq != tp->rcv_nxt) { 4829 struct sk_buff *skb = skb_peek(&sk->sk_receive_queue); 4830 tp->copied_seq++; 4831 if (skb && !before(tp->copied_seq, TCP_SKB_CB(skb)->end_seq)) { 4832 __skb_unlink(skb, &sk->sk_receive_queue); 4833 __kfree_skb(skb); 4834 } 4835 } 4836 4837 tp->urg_data = TCP_URG_NOTYET; 4838 tp->urg_seq = ptr; 4839 4840 /* Disable header prediction. */ 4841 tp->pred_flags = 0; 4842 } 4843 4844 /* This is the 'fast' part of urgent handling. */ 4845 static void tcp_urg(struct sock *sk, struct sk_buff *skb, const struct tcphdr *th) 4846 { 4847 struct tcp_sock *tp = tcp_sk(sk); 4848 4849 /* Check if we get a new urgent pointer - normally not. */ 4850 if (th->urg) 4851 tcp_check_urg(sk, th); 4852 4853 /* Do we wait for any urgent data? - normally not... */ 4854 if (tp->urg_data == TCP_URG_NOTYET) { 4855 u32 ptr = tp->urg_seq - ntohl(th->seq) + (th->doff * 4) - 4856 th->syn; 4857 4858 /* Is the urgent pointer pointing into this packet? */ 4859 if (ptr < skb->len) { 4860 u8 tmp; 4861 if (skb_copy_bits(skb, ptr, &tmp, 1)) 4862 BUG(); 4863 tp->urg_data = TCP_URG_VALID | tmp; 4864 if (!sock_flag(sk, SOCK_DEAD)) 4865 sk->sk_data_ready(sk, 0); 4866 } 4867 } 4868 } 4869 4870 static int tcp_copy_to_iovec(struct sock *sk, struct sk_buff *skb, int hlen) 4871 { 4872 struct tcp_sock *tp = tcp_sk(sk); 4873 int chunk = skb->len - hlen; 4874 int err; 4875 4876 local_bh_enable(); 4877 if (skb_csum_unnecessary(skb)) 4878 err = skb_copy_datagram_iovec(skb, hlen, tp->ucopy.iov, chunk); 4879 else 4880 err = skb_copy_and_csum_datagram_iovec(skb, hlen, 4881 tp->ucopy.iov); 4882 4883 if (!err) { 4884 tp->ucopy.len -= chunk; 4885 tp->copied_seq += chunk; 4886 tcp_rcv_space_adjust(sk); 4887 } 4888 4889 local_bh_disable(); 4890 return err; 4891 } 4892 4893 static __sum16 __tcp_checksum_complete_user(struct sock *sk, 4894 struct sk_buff *skb) 4895 { 4896 __sum16 result; 4897 4898 if (sock_owned_by_user(sk)) { 4899 local_bh_enable(); 4900 result = __tcp_checksum_complete(skb); 4901 local_bh_disable(); 4902 } else { 4903 result = __tcp_checksum_complete(skb); 4904 } 4905 return result; 4906 } 4907 4908 static inline bool tcp_checksum_complete_user(struct sock *sk, 4909 struct sk_buff *skb) 4910 { 4911 return !skb_csum_unnecessary(skb) && 4912 __tcp_checksum_complete_user(sk, skb); 4913 } 4914 4915 #ifdef CONFIG_NET_DMA 4916 static bool tcp_dma_try_early_copy(struct sock *sk, struct sk_buff *skb, 4917 int hlen) 4918 { 4919 struct tcp_sock *tp = tcp_sk(sk); 4920 int chunk = skb->len - hlen; 4921 int dma_cookie; 4922 bool copied_early = false; 4923 4924 if (tp->ucopy.wakeup) 4925 return false; 4926 4927 if (!tp->ucopy.dma_chan && tp->ucopy.pinned_list) 4928 tp->ucopy.dma_chan = net_dma_find_channel(); 4929 4930 if (tp->ucopy.dma_chan && skb_csum_unnecessary(skb)) { 4931 4932 dma_cookie = dma_skb_copy_datagram_iovec(tp->ucopy.dma_chan, 4933 skb, hlen, 4934 tp->ucopy.iov, chunk, 4935 tp->ucopy.pinned_list); 4936 4937 if (dma_cookie < 0) 4938 goto out; 4939 4940 tp->ucopy.dma_cookie = dma_cookie; 4941 copied_early = true; 4942 4943 tp->ucopy.len -= chunk; 4944 tp->copied_seq += chunk; 4945 tcp_rcv_space_adjust(sk); 4946 4947 if ((tp->ucopy.len == 0) || 4948 (tcp_flag_word(tcp_hdr(skb)) & TCP_FLAG_PSH) || 4949 (atomic_read(&sk->sk_rmem_alloc) > (sk->sk_rcvbuf >> 1))) { 4950 tp->ucopy.wakeup = 1; 4951 sk->sk_data_ready(sk, 0); 4952 } 4953 } else if (chunk > 0) { 4954 tp->ucopy.wakeup = 1; 4955 sk->sk_data_ready(sk, 0); 4956 } 4957 out: 4958 return copied_early; 4959 } 4960 #endif /* CONFIG_NET_DMA */ 4961 4962 /* Does PAWS and seqno based validation of an incoming segment, flags will 4963 * play significant role here. 4964 */ 4965 static bool tcp_validate_incoming(struct sock *sk, struct sk_buff *skb, 4966 const struct tcphdr *th, int syn_inerr) 4967 { 4968 struct tcp_sock *tp = tcp_sk(sk); 4969 4970 /* RFC1323: H1. Apply PAWS check first. */ 4971 if (tcp_fast_parse_options(skb, th, tp) && tp->rx_opt.saw_tstamp && 4972 tcp_paws_discard(sk, skb)) { 4973 if (!th->rst) { 4974 NET_INC_STATS_BH(sock_net(sk), LINUX_MIB_PAWSESTABREJECTED); 4975 tcp_send_dupack(sk, skb); 4976 goto discard; 4977 } 4978 /* Reset is accepted even if it did not pass PAWS. */ 4979 } 4980 4981 /* Step 1: check sequence number */ 4982 if (!tcp_sequence(tp, TCP_SKB_CB(skb)->seq, TCP_SKB_CB(skb)->end_seq)) { 4983 /* RFC793, page 37: "In all states except SYN-SENT, all reset 4984 * (RST) segments are validated by checking their SEQ-fields." 4985 * And page 69: "If an incoming segment is not acceptable, 4986 * an acknowledgment should be sent in reply (unless the RST 4987 * bit is set, if so drop the segment and return)". 4988 */ 4989 if (!th->rst) { 4990 if (th->syn) 4991 goto syn_challenge; 4992 tcp_send_dupack(sk, skb); 4993 } 4994 goto discard; 4995 } 4996 4997 /* Step 2: check RST bit */ 4998 if (th->rst) { 4999 /* RFC 5961 3.2 : 5000 * If sequence number exactly matches RCV.NXT, then 5001 * RESET the connection 5002 * else 5003 * Send a challenge ACK 5004 */ 5005 if (TCP_SKB_CB(skb)->seq == tp->rcv_nxt) 5006 tcp_reset(sk); 5007 else 5008 tcp_send_challenge_ack(sk); 5009 goto discard; 5010 } 5011 5012 /* step 3: check security and precedence [ignored] */ 5013 5014 /* step 4: Check for a SYN 5015 * RFC 5691 4.2 : Send a challenge ack 5016 */ 5017 if (th->syn) { 5018 syn_challenge: 5019 if (syn_inerr) 5020 TCP_INC_STATS_BH(sock_net(sk), TCP_MIB_INERRS); 5021 NET_INC_STATS_BH(sock_net(sk), LINUX_MIB_TCPSYNCHALLENGE); 5022 tcp_send_challenge_ack(sk); 5023 goto discard; 5024 } 5025 5026 return true; 5027 5028 discard: 5029 __kfree_skb(skb); 5030 return false; 5031 } 5032 5033 /* 5034 * TCP receive function for the ESTABLISHED state. 5035 * 5036 * It is split into a fast path and a slow path. The fast path is 5037 * disabled when: 5038 * - A zero window was announced from us - zero window probing 5039 * is only handled properly in the slow path. 5040 * - Out of order segments arrived. 5041 * - Urgent data is expected. 5042 * - There is no buffer space left 5043 * - Unexpected TCP flags/window values/header lengths are received 5044 * (detected by checking the TCP header against pred_flags) 5045 * - Data is sent in both directions. Fast path only supports pure senders 5046 * or pure receivers (this means either the sequence number or the ack 5047 * value must stay constant) 5048 * - Unexpected TCP option. 5049 * 5050 * When these conditions are not satisfied it drops into a standard 5051 * receive procedure patterned after RFC793 to handle all cases. 5052 * The first three cases are guaranteed by proper pred_flags setting, 5053 * the rest is checked inline. Fast processing is turned on in 5054 * tcp_data_queue when everything is OK. 5055 */ 5056 void tcp_rcv_established(struct sock *sk, struct sk_buff *skb, 5057 const struct tcphdr *th, unsigned int len) 5058 { 5059 struct tcp_sock *tp = tcp_sk(sk); 5060 5061 if (unlikely(sk->sk_rx_dst == NULL)) 5062 inet_csk(sk)->icsk_af_ops->sk_rx_dst_set(sk, skb); 5063 /* 5064 * Header prediction. 5065 * The code loosely follows the one in the famous 5066 * "30 instruction TCP receive" Van Jacobson mail. 5067 * 5068 * Van's trick is to deposit buffers into socket queue 5069 * on a device interrupt, to call tcp_recv function 5070 * on the receive process context and checksum and copy 5071 * the buffer to user space. smart... 5072 * 5073 * Our current scheme is not silly either but we take the 5074 * extra cost of the net_bh soft interrupt processing... 5075 * We do checksum and copy also but from device to kernel. 5076 */ 5077 5078 tp->rx_opt.saw_tstamp = 0; 5079 5080 /* pred_flags is 0xS?10 << 16 + snd_wnd 5081 * if header_prediction is to be made 5082 * 'S' will always be tp->tcp_header_len >> 2 5083 * '?' will be 0 for the fast path, otherwise pred_flags is 0 to 5084 * turn it off (when there are holes in the receive 5085 * space for instance) 5086 * PSH flag is ignored. 5087 */ 5088 5089 if ((tcp_flag_word(th) & TCP_HP_BITS) == tp->pred_flags && 5090 TCP_SKB_CB(skb)->seq == tp->rcv_nxt && 5091 !after(TCP_SKB_CB(skb)->ack_seq, tp->snd_nxt)) { 5092 int tcp_header_len = tp->tcp_header_len; 5093 5094 /* Timestamp header prediction: tcp_header_len 5095 * is automatically equal to th->doff*4 due to pred_flags 5096 * match. 5097 */ 5098 5099 /* Check timestamp */ 5100 if (tcp_header_len == sizeof(struct tcphdr) + TCPOLEN_TSTAMP_ALIGNED) { 5101 /* No? Slow path! */ 5102 if (!tcp_parse_aligned_timestamp(tp, th)) 5103 goto slow_path; 5104 5105 /* If PAWS failed, check it more carefully in slow path */ 5106 if ((s32)(tp->rx_opt.rcv_tsval - tp->rx_opt.ts_recent) < 0) 5107 goto slow_path; 5108 5109 /* DO NOT update ts_recent here, if checksum fails 5110 * and timestamp was corrupted part, it will result 5111 * in a hung connection since we will drop all 5112 * future packets due to the PAWS test. 5113 */ 5114 } 5115 5116 if (len <= tcp_header_len) { 5117 /* Bulk data transfer: sender */ 5118 if (len == tcp_header_len) { 5119 /* Predicted packet is in window by definition. 5120 * seq == rcv_nxt and rcv_wup <= rcv_nxt. 5121 * Hence, check seq<=rcv_wup reduces to: 5122 */ 5123 if (tcp_header_len == 5124 (sizeof(struct tcphdr) + TCPOLEN_TSTAMP_ALIGNED) && 5125 tp->rcv_nxt == tp->rcv_wup) 5126 tcp_store_ts_recent(tp); 5127 5128 /* We know that such packets are checksummed 5129 * on entry. 5130 */ 5131 tcp_ack(sk, skb, 0); 5132 __kfree_skb(skb); 5133 tcp_data_snd_check(sk); 5134 return; 5135 } else { /* Header too small */ 5136 TCP_INC_STATS_BH(sock_net(sk), TCP_MIB_INERRS); 5137 goto discard; 5138 } 5139 } else { 5140 int eaten = 0; 5141 int copied_early = 0; 5142 bool fragstolen = false; 5143 5144 if (tp->copied_seq == tp->rcv_nxt && 5145 len - tcp_header_len <= tp->ucopy.len) { 5146 #ifdef CONFIG_NET_DMA 5147 if (tp->ucopy.task == current && 5148 sock_owned_by_user(sk) && 5149 tcp_dma_try_early_copy(sk, skb, tcp_header_len)) { 5150 copied_early = 1; 5151 eaten = 1; 5152 } 5153 #endif 5154 if (tp->ucopy.task == current && 5155 sock_owned_by_user(sk) && !copied_early) { 5156 __set_current_state(TASK_RUNNING); 5157 5158 if (!tcp_copy_to_iovec(sk, skb, tcp_header_len)) 5159 eaten = 1; 5160 } 5161 if (eaten) { 5162 /* Predicted packet is in window by definition. 5163 * seq == rcv_nxt and rcv_wup <= rcv_nxt. 5164 * Hence, check seq<=rcv_wup reduces to: 5165 */ 5166 if (tcp_header_len == 5167 (sizeof(struct tcphdr) + 5168 TCPOLEN_TSTAMP_ALIGNED) && 5169 tp->rcv_nxt == tp->rcv_wup) 5170 tcp_store_ts_recent(tp); 5171 5172 tcp_rcv_rtt_measure_ts(sk, skb); 5173 5174 __skb_pull(skb, tcp_header_len); 5175 tp->rcv_nxt = TCP_SKB_CB(skb)->end_seq; 5176 NET_INC_STATS_BH(sock_net(sk), LINUX_MIB_TCPHPHITSTOUSER); 5177 } 5178 if (copied_early) 5179 tcp_cleanup_rbuf(sk, skb->len); 5180 } 5181 if (!eaten) { 5182 if (tcp_checksum_complete_user(sk, skb)) 5183 goto csum_error; 5184 5185 if ((int)skb->truesize > sk->sk_forward_alloc) 5186 goto step5; 5187 5188 /* Predicted packet is in window by definition. 5189 * seq == rcv_nxt and rcv_wup <= rcv_nxt. 5190 * Hence, check seq<=rcv_wup reduces to: 5191 */ 5192 if (tcp_header_len == 5193 (sizeof(struct tcphdr) + TCPOLEN_TSTAMP_ALIGNED) && 5194 tp->rcv_nxt == tp->rcv_wup) 5195 tcp_store_ts_recent(tp); 5196 5197 tcp_rcv_rtt_measure_ts(sk, skb); 5198 5199 NET_INC_STATS_BH(sock_net(sk), LINUX_MIB_TCPHPHITS); 5200 5201 /* Bulk data transfer: receiver */ 5202 eaten = tcp_queue_rcv(sk, skb, tcp_header_len, 5203 &fragstolen); 5204 } 5205 5206 tcp_event_data_recv(sk, skb); 5207 5208 if (TCP_SKB_CB(skb)->ack_seq != tp->snd_una) { 5209 /* Well, only one small jumplet in fast path... */ 5210 tcp_ack(sk, skb, FLAG_DATA); 5211 tcp_data_snd_check(sk); 5212 if (!inet_csk_ack_scheduled(sk)) 5213 goto no_ack; 5214 } 5215 5216 if (!copied_early || tp->rcv_nxt != tp->rcv_wup) 5217 __tcp_ack_snd_check(sk, 0); 5218 no_ack: 5219 #ifdef CONFIG_NET_DMA 5220 if (copied_early) 5221 __skb_queue_tail(&sk->sk_async_wait_queue, skb); 5222 else 5223 #endif 5224 if (eaten) 5225 kfree_skb_partial(skb, fragstolen); 5226 sk->sk_data_ready(sk, 0); 5227 return; 5228 } 5229 } 5230 5231 slow_path: 5232 if (len < (th->doff << 2) || tcp_checksum_complete_user(sk, skb)) 5233 goto csum_error; 5234 5235 if (!th->ack && !th->rst) 5236 goto discard; 5237 5238 /* 5239 * Standard slow path. 5240 */ 5241 5242 if (!tcp_validate_incoming(sk, skb, th, 1)) 5243 return; 5244 5245 step5: 5246 if (tcp_ack(sk, skb, FLAG_SLOWPATH | FLAG_UPDATE_TS_RECENT) < 0) 5247 goto discard; 5248 5249 tcp_rcv_rtt_measure_ts(sk, skb); 5250 5251 /* Process urgent data. */ 5252 tcp_urg(sk, skb, th); 5253 5254 /* step 7: process the segment text */ 5255 tcp_data_queue(sk, skb); 5256 5257 tcp_data_snd_check(sk); 5258 tcp_ack_snd_check(sk); 5259 return; 5260 5261 csum_error: 5262 TCP_INC_STATS_BH(sock_net(sk), TCP_MIB_CSUMERRORS); 5263 TCP_INC_STATS_BH(sock_net(sk), TCP_MIB_INERRS); 5264 5265 discard: 5266 __kfree_skb(skb); 5267 } 5268 EXPORT_SYMBOL(tcp_rcv_established); 5269 5270 void tcp_finish_connect(struct sock *sk, struct sk_buff *skb) 5271 { 5272 struct tcp_sock *tp = tcp_sk(sk); 5273 struct inet_connection_sock *icsk = inet_csk(sk); 5274 5275 tcp_set_state(sk, TCP_ESTABLISHED); 5276 5277 if (skb != NULL) { 5278 icsk->icsk_af_ops->sk_rx_dst_set(sk, skb); 5279 security_inet_conn_established(sk, skb); 5280 } 5281 5282 /* Make sure socket is routed, for correct metrics. */ 5283 icsk->icsk_af_ops->rebuild_header(sk); 5284 5285 tcp_init_metrics(sk); 5286 5287 tcp_init_congestion_control(sk); 5288 5289 /* Prevent spurious tcp_cwnd_restart() on first data 5290 * packet. 5291 */ 5292 tp->lsndtime = tcp_time_stamp; 5293 5294 tcp_init_buffer_space(sk); 5295 5296 if (sock_flag(sk, SOCK_KEEPOPEN)) 5297 inet_csk_reset_keepalive_timer(sk, keepalive_time_when(tp)); 5298 5299 if (!tp->rx_opt.snd_wscale) 5300 __tcp_fast_path_on(tp, tp->snd_wnd); 5301 else 5302 tp->pred_flags = 0; 5303 5304 if (!sock_flag(sk, SOCK_DEAD)) { 5305 sk->sk_state_change(sk); 5306 sk_wake_async(sk, SOCK_WAKE_IO, POLL_OUT); 5307 } 5308 } 5309 5310 static bool tcp_rcv_fastopen_synack(struct sock *sk, struct sk_buff *synack, 5311 struct tcp_fastopen_cookie *cookie) 5312 { 5313 struct tcp_sock *tp = tcp_sk(sk); 5314 struct sk_buff *data = tp->syn_data ? tcp_write_queue_head(sk) : NULL; 5315 u16 mss = tp->rx_opt.mss_clamp; 5316 bool syn_drop; 5317 5318 if (mss == tp->rx_opt.user_mss) { 5319 struct tcp_options_received opt; 5320 5321 /* Get original SYNACK MSS value if user MSS sets mss_clamp */ 5322 tcp_clear_options(&opt); 5323 opt.user_mss = opt.mss_clamp = 0; 5324 tcp_parse_options(synack, &opt, 0, NULL); 5325 mss = opt.mss_clamp; 5326 } 5327 5328 if (!tp->syn_fastopen) /* Ignore an unsolicited cookie */ 5329 cookie->len = -1; 5330 5331 /* The SYN-ACK neither has cookie nor acknowledges the data. Presumably 5332 * the remote receives only the retransmitted (regular) SYNs: either 5333 * the original SYN-data or the corresponding SYN-ACK is lost. 5334 */ 5335 syn_drop = (cookie->len <= 0 && data && tp->total_retrans); 5336 5337 tcp_fastopen_cache_set(sk, mss, cookie, syn_drop); 5338 5339 if (data) { /* Retransmit unacked data in SYN */ 5340 tcp_for_write_queue_from(data, sk) { 5341 if (data == tcp_send_head(sk) || 5342 __tcp_retransmit_skb(sk, data)) 5343 break; 5344 } 5345 tcp_rearm_rto(sk); 5346 return true; 5347 } 5348 tp->syn_data_acked = tp->syn_data; 5349 return false; 5350 } 5351 5352 static int tcp_rcv_synsent_state_process(struct sock *sk, struct sk_buff *skb, 5353 const struct tcphdr *th, unsigned int len) 5354 { 5355 struct inet_connection_sock *icsk = inet_csk(sk); 5356 struct tcp_sock *tp = tcp_sk(sk); 5357 struct tcp_fastopen_cookie foc = { .len = -1 }; 5358 int saved_clamp = tp->rx_opt.mss_clamp; 5359 5360 tcp_parse_options(skb, &tp->rx_opt, 0, &foc); 5361 if (tp->rx_opt.saw_tstamp && tp->rx_opt.rcv_tsecr) 5362 tp->rx_opt.rcv_tsecr -= tp->tsoffset; 5363 5364 if (th->ack) { 5365 /* rfc793: 5366 * "If the state is SYN-SENT then 5367 * first check the ACK bit 5368 * If the ACK bit is set 5369 * If SEG.ACK =< ISS, or SEG.ACK > SND.NXT, send 5370 * a reset (unless the RST bit is set, if so drop 5371 * the segment and return)" 5372 */ 5373 if (!after(TCP_SKB_CB(skb)->ack_seq, tp->snd_una) || 5374 after(TCP_SKB_CB(skb)->ack_seq, tp->snd_nxt)) 5375 goto reset_and_undo; 5376 5377 if (tp->rx_opt.saw_tstamp && tp->rx_opt.rcv_tsecr && 5378 !between(tp->rx_opt.rcv_tsecr, tp->retrans_stamp, 5379 tcp_time_stamp)) { 5380 NET_INC_STATS_BH(sock_net(sk), LINUX_MIB_PAWSACTIVEREJECTED); 5381 goto reset_and_undo; 5382 } 5383 5384 /* Now ACK is acceptable. 5385 * 5386 * "If the RST bit is set 5387 * If the ACK was acceptable then signal the user "error: 5388 * connection reset", drop the segment, enter CLOSED state, 5389 * delete TCB, and return." 5390 */ 5391 5392 if (th->rst) { 5393 tcp_reset(sk); 5394 goto discard; 5395 } 5396 5397 /* rfc793: 5398 * "fifth, if neither of the SYN or RST bits is set then 5399 * drop the segment and return." 5400 * 5401 * See note below! 5402 * --ANK(990513) 5403 */ 5404 if (!th->syn) 5405 goto discard_and_undo; 5406 5407 /* rfc793: 5408 * "If the SYN bit is on ... 5409 * are acceptable then ... 5410 * (our SYN has been ACKed), change the connection 5411 * state to ESTABLISHED..." 5412 */ 5413 5414 TCP_ECN_rcv_synack(tp, th); 5415 5416 tcp_init_wl(tp, TCP_SKB_CB(skb)->seq); 5417 tcp_ack(sk, skb, FLAG_SLOWPATH); 5418 5419 /* Ok.. it's good. Set up sequence numbers and 5420 * move to established. 5421 */ 5422 tp->rcv_nxt = TCP_SKB_CB(skb)->seq + 1; 5423 tp->rcv_wup = TCP_SKB_CB(skb)->seq + 1; 5424 5425 /* RFC1323: The window in SYN & SYN/ACK segments is 5426 * never scaled. 5427 */ 5428 tp->snd_wnd = ntohs(th->window); 5429 5430 if (!tp->rx_opt.wscale_ok) { 5431 tp->rx_opt.snd_wscale = tp->rx_opt.rcv_wscale = 0; 5432 tp->window_clamp = min(tp->window_clamp, 65535U); 5433 } 5434 5435 if (tp->rx_opt.saw_tstamp) { 5436 tp->rx_opt.tstamp_ok = 1; 5437 tp->tcp_header_len = 5438 sizeof(struct tcphdr) + TCPOLEN_TSTAMP_ALIGNED; 5439 tp->advmss -= TCPOLEN_TSTAMP_ALIGNED; 5440 tcp_store_ts_recent(tp); 5441 } else { 5442 tp->tcp_header_len = sizeof(struct tcphdr); 5443 } 5444 5445 if (tcp_is_sack(tp) && sysctl_tcp_fack) 5446 tcp_enable_fack(tp); 5447 5448 tcp_mtup_init(sk); 5449 tcp_sync_mss(sk, icsk->icsk_pmtu_cookie); 5450 tcp_initialize_rcv_mss(sk); 5451 5452 /* Remember, tcp_poll() does not lock socket! 5453 * Change state from SYN-SENT only after copied_seq 5454 * is initialized. */ 5455 tp->copied_seq = tp->rcv_nxt; 5456 5457 smp_mb(); 5458 5459 tcp_finish_connect(sk, skb); 5460 5461 if ((tp->syn_fastopen || tp->syn_data) && 5462 tcp_rcv_fastopen_synack(sk, skb, &foc)) 5463 return -1; 5464 5465 if (sk->sk_write_pending || 5466 icsk->icsk_accept_queue.rskq_defer_accept || 5467 icsk->icsk_ack.pingpong) { 5468 /* Save one ACK. Data will be ready after 5469 * several ticks, if write_pending is set. 5470 * 5471 * It may be deleted, but with this feature tcpdumps 5472 * look so _wonderfully_ clever, that I was not able 5473 * to stand against the temptation 8) --ANK 5474 */ 5475 inet_csk_schedule_ack(sk); 5476 icsk->icsk_ack.lrcvtime = tcp_time_stamp; 5477 tcp_enter_quickack_mode(sk); 5478 inet_csk_reset_xmit_timer(sk, ICSK_TIME_DACK, 5479 TCP_DELACK_MAX, TCP_RTO_MAX); 5480 5481 discard: 5482 __kfree_skb(skb); 5483 return 0; 5484 } else { 5485 tcp_send_ack(sk); 5486 } 5487 return -1; 5488 } 5489 5490 /* No ACK in the segment */ 5491 5492 if (th->rst) { 5493 /* rfc793: 5494 * "If the RST bit is set 5495 * 5496 * Otherwise (no ACK) drop the segment and return." 5497 */ 5498 5499 goto discard_and_undo; 5500 } 5501 5502 /* PAWS check. */ 5503 if (tp->rx_opt.ts_recent_stamp && tp->rx_opt.saw_tstamp && 5504 tcp_paws_reject(&tp->rx_opt, 0)) 5505 goto discard_and_undo; 5506 5507 if (th->syn) { 5508 /* We see SYN without ACK. It is attempt of 5509 * simultaneous connect with crossed SYNs. 5510 * Particularly, it can be connect to self. 5511 */ 5512 tcp_set_state(sk, TCP_SYN_RECV); 5513 5514 if (tp->rx_opt.saw_tstamp) { 5515 tp->rx_opt.tstamp_ok = 1; 5516 tcp_store_ts_recent(tp); 5517 tp->tcp_header_len = 5518 sizeof(struct tcphdr) + TCPOLEN_TSTAMP_ALIGNED; 5519 } else { 5520 tp->tcp_header_len = sizeof(struct tcphdr); 5521 } 5522 5523 tp->rcv_nxt = TCP_SKB_CB(skb)->seq + 1; 5524 tp->rcv_wup = TCP_SKB_CB(skb)->seq + 1; 5525 5526 /* RFC1323: The window in SYN & SYN/ACK segments is 5527 * never scaled. 5528 */ 5529 tp->snd_wnd = ntohs(th->window); 5530 tp->snd_wl1 = TCP_SKB_CB(skb)->seq; 5531 tp->max_window = tp->snd_wnd; 5532 5533 TCP_ECN_rcv_syn(tp, th); 5534 5535 tcp_mtup_init(sk); 5536 tcp_sync_mss(sk, icsk->icsk_pmtu_cookie); 5537 tcp_initialize_rcv_mss(sk); 5538 5539 tcp_send_synack(sk); 5540 #if 0 5541 /* Note, we could accept data and URG from this segment. 5542 * There are no obstacles to make this (except that we must 5543 * either change tcp_recvmsg() to prevent it from returning data 5544 * before 3WHS completes per RFC793, or employ TCP Fast Open). 5545 * 5546 * However, if we ignore data in ACKless segments sometimes, 5547 * we have no reasons to accept it sometimes. 5548 * Also, seems the code doing it in step6 of tcp_rcv_state_process 5549 * is not flawless. So, discard packet for sanity. 5550 * Uncomment this return to process the data. 5551 */ 5552 return -1; 5553 #else 5554 goto discard; 5555 #endif 5556 } 5557 /* "fifth, if neither of the SYN or RST bits is set then 5558 * drop the segment and return." 5559 */ 5560 5561 discard_and_undo: 5562 tcp_clear_options(&tp->rx_opt); 5563 tp->rx_opt.mss_clamp = saved_clamp; 5564 goto discard; 5565 5566 reset_and_undo: 5567 tcp_clear_options(&tp->rx_opt); 5568 tp->rx_opt.mss_clamp = saved_clamp; 5569 return 1; 5570 } 5571 5572 /* 5573 * This function implements the receiving procedure of RFC 793 for 5574 * all states except ESTABLISHED and TIME_WAIT. 5575 * It's called from both tcp_v4_rcv and tcp_v6_rcv and should be 5576 * address independent. 5577 */ 5578 5579 int tcp_rcv_state_process(struct sock *sk, struct sk_buff *skb, 5580 const struct tcphdr *th, unsigned int len) 5581 { 5582 struct tcp_sock *tp = tcp_sk(sk); 5583 struct inet_connection_sock *icsk = inet_csk(sk); 5584 struct request_sock *req; 5585 int queued = 0; 5586 bool acceptable; 5587 5588 tp->rx_opt.saw_tstamp = 0; 5589 5590 switch (sk->sk_state) { 5591 case TCP_CLOSE: 5592 goto discard; 5593 5594 case TCP_LISTEN: 5595 if (th->ack) 5596 return 1; 5597 5598 if (th->rst) 5599 goto discard; 5600 5601 if (th->syn) { 5602 if (th->fin) 5603 goto discard; 5604 if (icsk->icsk_af_ops->conn_request(sk, skb) < 0) 5605 return 1; 5606 5607 /* Now we have several options: In theory there is 5608 * nothing else in the frame. KA9Q has an option to 5609 * send data with the syn, BSD accepts data with the 5610 * syn up to the [to be] advertised window and 5611 * Solaris 2.1 gives you a protocol error. For now 5612 * we just ignore it, that fits the spec precisely 5613 * and avoids incompatibilities. It would be nice in 5614 * future to drop through and process the data. 5615 * 5616 * Now that TTCP is starting to be used we ought to 5617 * queue this data. 5618 * But, this leaves one open to an easy denial of 5619 * service attack, and SYN cookies can't defend 5620 * against this problem. So, we drop the data 5621 * in the interest of security over speed unless 5622 * it's still in use. 5623 */ 5624 kfree_skb(skb); 5625 return 0; 5626 } 5627 goto discard; 5628 5629 case TCP_SYN_SENT: 5630 queued = tcp_rcv_synsent_state_process(sk, skb, th, len); 5631 if (queued >= 0) 5632 return queued; 5633 5634 /* Do step6 onward by hand. */ 5635 tcp_urg(sk, skb, th); 5636 __kfree_skb(skb); 5637 tcp_data_snd_check(sk); 5638 return 0; 5639 } 5640 5641 req = tp->fastopen_rsk; 5642 if (req != NULL) { 5643 WARN_ON_ONCE(sk->sk_state != TCP_SYN_RECV && 5644 sk->sk_state != TCP_FIN_WAIT1); 5645 5646 if (tcp_check_req(sk, skb, req, NULL, true) == NULL) 5647 goto discard; 5648 } 5649 5650 if (!th->ack && !th->rst) 5651 goto discard; 5652 5653 if (!tcp_validate_incoming(sk, skb, th, 0)) 5654 return 0; 5655 5656 /* step 5: check the ACK field */ 5657 acceptable = tcp_ack(sk, skb, FLAG_SLOWPATH | 5658 FLAG_UPDATE_TS_RECENT) > 0; 5659 5660 switch (sk->sk_state) { 5661 case TCP_SYN_RECV: 5662 if (!acceptable) 5663 return 1; 5664 5665 /* Once we leave TCP_SYN_RECV, we no longer need req 5666 * so release it. 5667 */ 5668 if (req) { 5669 tp->total_retrans = req->num_retrans; 5670 reqsk_fastopen_remove(sk, req, false); 5671 } else { 5672 /* Make sure socket is routed, for correct metrics. */ 5673 icsk->icsk_af_ops->rebuild_header(sk); 5674 tcp_init_congestion_control(sk); 5675 5676 tcp_mtup_init(sk); 5677 tcp_init_buffer_space(sk); 5678 tp->copied_seq = tp->rcv_nxt; 5679 } 5680 smp_mb(); 5681 tcp_set_state(sk, TCP_ESTABLISHED); 5682 sk->sk_state_change(sk); 5683 5684 /* Note, that this wakeup is only for marginal crossed SYN case. 5685 * Passively open sockets are not waked up, because 5686 * sk->sk_sleep == NULL and sk->sk_socket == NULL. 5687 */ 5688 if (sk->sk_socket) 5689 sk_wake_async(sk, SOCK_WAKE_IO, POLL_OUT); 5690 5691 tp->snd_una = TCP_SKB_CB(skb)->ack_seq; 5692 tp->snd_wnd = ntohs(th->window) << tp->rx_opt.snd_wscale; 5693 tcp_init_wl(tp, TCP_SKB_CB(skb)->seq); 5694 tcp_synack_rtt_meas(sk, req); 5695 5696 if (tp->rx_opt.tstamp_ok) 5697 tp->advmss -= TCPOLEN_TSTAMP_ALIGNED; 5698 5699 if (req) { 5700 /* Re-arm the timer because data may have been sent out. 5701 * This is similar to the regular data transmission case 5702 * when new data has just been ack'ed. 5703 * 5704 * (TFO) - we could try to be more aggressive and 5705 * retransmitting any data sooner based on when they 5706 * are sent out. 5707 */ 5708 tcp_rearm_rto(sk); 5709 } else 5710 tcp_init_metrics(sk); 5711 5712 /* Prevent spurious tcp_cwnd_restart() on first data packet */ 5713 tp->lsndtime = tcp_time_stamp; 5714 5715 tcp_initialize_rcv_mss(sk); 5716 tcp_fast_path_on(tp); 5717 break; 5718 5719 case TCP_FIN_WAIT1: { 5720 struct dst_entry *dst; 5721 int tmo; 5722 5723 /* If we enter the TCP_FIN_WAIT1 state and we are a 5724 * Fast Open socket and this is the first acceptable 5725 * ACK we have received, this would have acknowledged 5726 * our SYNACK so stop the SYNACK timer. 5727 */ 5728 if (req != NULL) { 5729 /* Return RST if ack_seq is invalid. 5730 * Note that RFC793 only says to generate a 5731 * DUPACK for it but for TCP Fast Open it seems 5732 * better to treat this case like TCP_SYN_RECV 5733 * above. 5734 */ 5735 if (!acceptable) 5736 return 1; 5737 /* We no longer need the request sock. */ 5738 reqsk_fastopen_remove(sk, req, false); 5739 tcp_rearm_rto(sk); 5740 } 5741 if (tp->snd_una != tp->write_seq) 5742 break; 5743 5744 tcp_set_state(sk, TCP_FIN_WAIT2); 5745 sk->sk_shutdown |= SEND_SHUTDOWN; 5746 5747 dst = __sk_dst_get(sk); 5748 if (dst) 5749 dst_confirm(dst); 5750 5751 if (!sock_flag(sk, SOCK_DEAD)) { 5752 /* Wake up lingering close() */ 5753 sk->sk_state_change(sk); 5754 break; 5755 } 5756 5757 if (tp->linger2 < 0 || 5758 (TCP_SKB_CB(skb)->end_seq != TCP_SKB_CB(skb)->seq && 5759 after(TCP_SKB_CB(skb)->end_seq - th->fin, tp->rcv_nxt))) { 5760 tcp_done(sk); 5761 NET_INC_STATS_BH(sock_net(sk), LINUX_MIB_TCPABORTONDATA); 5762 return 1; 5763 } 5764 5765 tmo = tcp_fin_time(sk); 5766 if (tmo > TCP_TIMEWAIT_LEN) { 5767 inet_csk_reset_keepalive_timer(sk, tmo - TCP_TIMEWAIT_LEN); 5768 } else if (th->fin || sock_owned_by_user(sk)) { 5769 /* Bad case. We could lose such FIN otherwise. 5770 * It is not a big problem, but it looks confusing 5771 * and not so rare event. We still can lose it now, 5772 * if it spins in bh_lock_sock(), but it is really 5773 * marginal case. 5774 */ 5775 inet_csk_reset_keepalive_timer(sk, tmo); 5776 } else { 5777 tcp_time_wait(sk, TCP_FIN_WAIT2, tmo); 5778 goto discard; 5779 } 5780 break; 5781 } 5782 5783 case TCP_CLOSING: 5784 if (tp->snd_una == tp->write_seq) { 5785 tcp_time_wait(sk, TCP_TIME_WAIT, 0); 5786 goto discard; 5787 } 5788 break; 5789 5790 case TCP_LAST_ACK: 5791 if (tp->snd_una == tp->write_seq) { 5792 tcp_update_metrics(sk); 5793 tcp_done(sk); 5794 goto discard; 5795 } 5796 break; 5797 } 5798 5799 /* step 6: check the URG bit */ 5800 tcp_urg(sk, skb, th); 5801 5802 /* step 7: process the segment text */ 5803 switch (sk->sk_state) { 5804 case TCP_CLOSE_WAIT: 5805 case TCP_CLOSING: 5806 case TCP_LAST_ACK: 5807 if (!before(TCP_SKB_CB(skb)->seq, tp->rcv_nxt)) 5808 break; 5809 case TCP_FIN_WAIT1: 5810 case TCP_FIN_WAIT2: 5811 /* RFC 793 says to queue data in these states, 5812 * RFC 1122 says we MUST send a reset. 5813 * BSD 4.4 also does reset. 5814 */ 5815 if (sk->sk_shutdown & RCV_SHUTDOWN) { 5816 if (TCP_SKB_CB(skb)->end_seq != TCP_SKB_CB(skb)->seq && 5817 after(TCP_SKB_CB(skb)->end_seq - th->fin, tp->rcv_nxt)) { 5818 NET_INC_STATS_BH(sock_net(sk), LINUX_MIB_TCPABORTONDATA); 5819 tcp_reset(sk); 5820 return 1; 5821 } 5822 } 5823 /* Fall through */ 5824 case TCP_ESTABLISHED: 5825 tcp_data_queue(sk, skb); 5826 queued = 1; 5827 break; 5828 } 5829 5830 /* tcp_data could move socket to TIME-WAIT */ 5831 if (sk->sk_state != TCP_CLOSE) { 5832 tcp_data_snd_check(sk); 5833 tcp_ack_snd_check(sk); 5834 } 5835 5836 if (!queued) { 5837 discard: 5838 __kfree_skb(skb); 5839 } 5840 return 0; 5841 } 5842 EXPORT_SYMBOL(tcp_rcv_state_process); 5843