1 /* 2 * INET An implementation of the TCP/IP protocol suite for the LINUX 3 * operating system. INET is implemented using the BSD Socket 4 * interface as the means of communication with the user level. 5 * 6 * Implementation of the Transmission Control Protocol(TCP). 7 * 8 * Authors: Ross Biro 9 * Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG> 10 * Mark Evans, <evansmp@uhura.aston.ac.uk> 11 * Corey Minyard <wf-rch!minyard@relay.EU.net> 12 * Florian La Roche, <flla@stud.uni-sb.de> 13 * Charles Hedrick, <hedrick@klinzhai.rutgers.edu> 14 * Linus Torvalds, <torvalds@cs.helsinki.fi> 15 * Alan Cox, <gw4pts@gw4pts.ampr.org> 16 * Matthew Dillon, <dillon@apollo.west.oic.com> 17 * Arnt Gulbrandsen, <agulbra@nvg.unit.no> 18 * Jorge Cwik, <jorge@laser.satlink.net> 19 */ 20 21 /* 22 * Changes: Pedro Roque : Retransmit queue handled by TCP. 23 * : Fragmentation on mtu decrease 24 * : Segment collapse on retransmit 25 * : AF independence 26 * 27 * Linus Torvalds : send_delayed_ack 28 * David S. Miller : Charge memory using the right skb 29 * during syn/ack processing. 30 * David S. Miller : Output engine completely rewritten. 31 * Andrea Arcangeli: SYNACK carry ts_recent in tsecr. 32 * Cacophonix Gaul : draft-minshall-nagle-01 33 * J Hadi Salim : ECN support 34 * 35 */ 36 37 #define pr_fmt(fmt) "TCP: " fmt 38 39 #include <net/tcp.h> 40 41 #include <linux/compiler.h> 42 #include <linux/gfp.h> 43 #include <linux/module.h> 44 45 /* People can turn this off for buggy TCP's found in printers etc. */ 46 int sysctl_tcp_retrans_collapse __read_mostly = 1; 47 48 /* People can turn this on to work with those rare, broken TCPs that 49 * interpret the window field as a signed quantity. 50 */ 51 int sysctl_tcp_workaround_signed_windows __read_mostly = 0; 52 53 /* Default TSQ limit of four TSO segments */ 54 int sysctl_tcp_limit_output_bytes __read_mostly = 262144; 55 56 /* This limits the percentage of the congestion window which we 57 * will allow a single TSO frame to consume. Building TSO frames 58 * which are too large can cause TCP streams to be bursty. 59 */ 60 int sysctl_tcp_tso_win_divisor __read_mostly = 3; 61 62 /* By default, RFC2861 behavior. */ 63 int sysctl_tcp_slow_start_after_idle __read_mostly = 1; 64 65 static bool tcp_write_xmit(struct sock *sk, unsigned int mss_now, int nonagle, 66 int push_one, gfp_t gfp); 67 68 /* Account for new data that has been sent to the network. */ 69 static void tcp_event_new_data_sent(struct sock *sk, const struct sk_buff *skb) 70 { 71 struct inet_connection_sock *icsk = inet_csk(sk); 72 struct tcp_sock *tp = tcp_sk(sk); 73 unsigned int prior_packets = tp->packets_out; 74 75 tcp_advance_send_head(sk, skb); 76 tp->snd_nxt = TCP_SKB_CB(skb)->end_seq; 77 78 tp->packets_out += tcp_skb_pcount(skb); 79 if (!prior_packets || icsk->icsk_pending == ICSK_TIME_LOSS_PROBE) 80 tcp_rearm_rto(sk); 81 82 NET_ADD_STATS(sock_net(sk), LINUX_MIB_TCPORIGDATASENT, 83 tcp_skb_pcount(skb)); 84 } 85 86 /* SND.NXT, if window was not shrunk or the amount of shrunk was less than one 87 * window scaling factor due to loss of precision. 88 * If window has been shrunk, what should we make? It is not clear at all. 89 * Using SND.UNA we will fail to open window, SND.NXT is out of window. :-( 90 * Anything in between SND.UNA...SND.UNA+SND.WND also can be already 91 * invalid. OK, let's make this for now: 92 */ 93 static inline __u32 tcp_acceptable_seq(const struct sock *sk) 94 { 95 const struct tcp_sock *tp = tcp_sk(sk); 96 97 if (!before(tcp_wnd_end(tp), tp->snd_nxt) || 98 (tp->rx_opt.wscale_ok && 99 ((tp->snd_nxt - tcp_wnd_end(tp)) < (1 << tp->rx_opt.rcv_wscale)))) 100 return tp->snd_nxt; 101 else 102 return tcp_wnd_end(tp); 103 } 104 105 /* Calculate mss to advertise in SYN segment. 106 * RFC1122, RFC1063, draft-ietf-tcpimpl-pmtud-01 state that: 107 * 108 * 1. It is independent of path mtu. 109 * 2. Ideally, it is maximal possible segment size i.e. 65535-40. 110 * 3. For IPv4 it is reasonable to calculate it from maximal MTU of 111 * attached devices, because some buggy hosts are confused by 112 * large MSS. 113 * 4. We do not make 3, we advertise MSS, calculated from first 114 * hop device mtu, but allow to raise it to ip_rt_min_advmss. 115 * This may be overridden via information stored in routing table. 116 * 5. Value 65535 for MSS is valid in IPv6 and means "as large as possible, 117 * probably even Jumbo". 118 */ 119 static __u16 tcp_advertise_mss(struct sock *sk) 120 { 121 struct tcp_sock *tp = tcp_sk(sk); 122 const struct dst_entry *dst = __sk_dst_get(sk); 123 int mss = tp->advmss; 124 125 if (dst) { 126 unsigned int metric = dst_metric_advmss(dst); 127 128 if (metric < mss) { 129 mss = metric; 130 tp->advmss = mss; 131 } 132 } 133 134 return (__u16)mss; 135 } 136 137 /* RFC2861. Reset CWND after idle period longer RTO to "restart window". 138 * This is the first part of cwnd validation mechanism. 139 */ 140 void tcp_cwnd_restart(struct sock *sk, s32 delta) 141 { 142 struct tcp_sock *tp = tcp_sk(sk); 143 u32 restart_cwnd = tcp_init_cwnd(tp, __sk_dst_get(sk)); 144 u32 cwnd = tp->snd_cwnd; 145 146 tcp_ca_event(sk, CA_EVENT_CWND_RESTART); 147 148 tp->snd_ssthresh = tcp_current_ssthresh(sk); 149 restart_cwnd = min(restart_cwnd, cwnd); 150 151 while ((delta -= inet_csk(sk)->icsk_rto) > 0 && cwnd > restart_cwnd) 152 cwnd >>= 1; 153 tp->snd_cwnd = max(cwnd, restart_cwnd); 154 tp->snd_cwnd_stamp = tcp_jiffies32; 155 tp->snd_cwnd_used = 0; 156 } 157 158 /* Congestion state accounting after a packet has been sent. */ 159 static void tcp_event_data_sent(struct tcp_sock *tp, 160 struct sock *sk) 161 { 162 struct inet_connection_sock *icsk = inet_csk(sk); 163 const u32 now = tcp_jiffies32; 164 165 if (tcp_packets_in_flight(tp) == 0) 166 tcp_ca_event(sk, CA_EVENT_TX_START); 167 168 tp->lsndtime = now; 169 170 /* If it is a reply for ato after last received 171 * packet, enter pingpong mode. 172 */ 173 if ((u32)(now - icsk->icsk_ack.lrcvtime) < icsk->icsk_ack.ato) 174 icsk->icsk_ack.pingpong = 1; 175 } 176 177 /* Account for an ACK we sent. */ 178 static inline void tcp_event_ack_sent(struct sock *sk, unsigned int pkts) 179 { 180 tcp_dec_quickack_mode(sk, pkts); 181 inet_csk_clear_xmit_timer(sk, ICSK_TIME_DACK); 182 } 183 184 185 u32 tcp_default_init_rwnd(u32 mss) 186 { 187 /* Initial receive window should be twice of TCP_INIT_CWND to 188 * enable proper sending of new unsent data during fast recovery 189 * (RFC 3517, Section 4, NextSeg() rule (2)). Further place a 190 * limit when mss is larger than 1460. 191 */ 192 u32 init_rwnd = TCP_INIT_CWND * 2; 193 194 if (mss > 1460) 195 init_rwnd = max((1460 * init_rwnd) / mss, 2U); 196 return init_rwnd; 197 } 198 199 /* Determine a window scaling and initial window to offer. 200 * Based on the assumption that the given amount of space 201 * will be offered. Store the results in the tp structure. 202 * NOTE: for smooth operation initial space offering should 203 * be a multiple of mss if possible. We assume here that mss >= 1. 204 * This MUST be enforced by all callers. 205 */ 206 void tcp_select_initial_window(int __space, __u32 mss, 207 __u32 *rcv_wnd, __u32 *window_clamp, 208 int wscale_ok, __u8 *rcv_wscale, 209 __u32 init_rcv_wnd) 210 { 211 unsigned int space = (__space < 0 ? 0 : __space); 212 213 /* If no clamp set the clamp to the max possible scaled window */ 214 if (*window_clamp == 0) 215 (*window_clamp) = (U16_MAX << TCP_MAX_WSCALE); 216 space = min(*window_clamp, space); 217 218 /* Quantize space offering to a multiple of mss if possible. */ 219 if (space > mss) 220 space = rounddown(space, mss); 221 222 /* NOTE: offering an initial window larger than 32767 223 * will break some buggy TCP stacks. If the admin tells us 224 * it is likely we could be speaking with such a buggy stack 225 * we will truncate our initial window offering to 32K-1 226 * unless the remote has sent us a window scaling option, 227 * which we interpret as a sign the remote TCP is not 228 * misinterpreting the window field as a signed quantity. 229 */ 230 if (sysctl_tcp_workaround_signed_windows) 231 (*rcv_wnd) = min(space, MAX_TCP_WINDOW); 232 else 233 (*rcv_wnd) = space; 234 235 (*rcv_wscale) = 0; 236 if (wscale_ok) { 237 /* Set window scaling on max possible window */ 238 space = max_t(u32, space, sysctl_tcp_rmem[2]); 239 space = max_t(u32, space, sysctl_rmem_max); 240 space = min_t(u32, space, *window_clamp); 241 while (space > U16_MAX && (*rcv_wscale) < TCP_MAX_WSCALE) { 242 space >>= 1; 243 (*rcv_wscale)++; 244 } 245 } 246 247 if (mss > (1 << *rcv_wscale)) { 248 if (!init_rcv_wnd) /* Use default unless specified otherwise */ 249 init_rcv_wnd = tcp_default_init_rwnd(mss); 250 *rcv_wnd = min(*rcv_wnd, init_rcv_wnd * mss); 251 } 252 253 /* Set the clamp no higher than max representable value */ 254 (*window_clamp) = min_t(__u32, U16_MAX << (*rcv_wscale), *window_clamp); 255 } 256 EXPORT_SYMBOL(tcp_select_initial_window); 257 258 /* Chose a new window to advertise, update state in tcp_sock for the 259 * socket, and return result with RFC1323 scaling applied. The return 260 * value can be stuffed directly into th->window for an outgoing 261 * frame. 262 */ 263 static u16 tcp_select_window(struct sock *sk) 264 { 265 struct tcp_sock *tp = tcp_sk(sk); 266 u32 old_win = tp->rcv_wnd; 267 u32 cur_win = tcp_receive_window(tp); 268 u32 new_win = __tcp_select_window(sk); 269 270 /* Never shrink the offered window */ 271 if (new_win < cur_win) { 272 /* Danger Will Robinson! 273 * Don't update rcv_wup/rcv_wnd here or else 274 * we will not be able to advertise a zero 275 * window in time. --DaveM 276 * 277 * Relax Will Robinson. 278 */ 279 if (new_win == 0) 280 NET_INC_STATS(sock_net(sk), 281 LINUX_MIB_TCPWANTZEROWINDOWADV); 282 new_win = ALIGN(cur_win, 1 << tp->rx_opt.rcv_wscale); 283 } 284 tp->rcv_wnd = new_win; 285 tp->rcv_wup = tp->rcv_nxt; 286 287 /* Make sure we do not exceed the maximum possible 288 * scaled window. 289 */ 290 if (!tp->rx_opt.rcv_wscale && sysctl_tcp_workaround_signed_windows) 291 new_win = min(new_win, MAX_TCP_WINDOW); 292 else 293 new_win = min(new_win, (65535U << tp->rx_opt.rcv_wscale)); 294 295 /* RFC1323 scaling applied */ 296 new_win >>= tp->rx_opt.rcv_wscale; 297 298 /* If we advertise zero window, disable fast path. */ 299 if (new_win == 0) { 300 tp->pred_flags = 0; 301 if (old_win) 302 NET_INC_STATS(sock_net(sk), 303 LINUX_MIB_TCPTOZEROWINDOWADV); 304 } else if (old_win == 0) { 305 NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPFROMZEROWINDOWADV); 306 } 307 308 return new_win; 309 } 310 311 /* Packet ECN state for a SYN-ACK */ 312 static void tcp_ecn_send_synack(struct sock *sk, struct sk_buff *skb) 313 { 314 const struct tcp_sock *tp = tcp_sk(sk); 315 316 TCP_SKB_CB(skb)->tcp_flags &= ~TCPHDR_CWR; 317 if (!(tp->ecn_flags & TCP_ECN_OK)) 318 TCP_SKB_CB(skb)->tcp_flags &= ~TCPHDR_ECE; 319 else if (tcp_ca_needs_ecn(sk) || 320 tcp_bpf_ca_needs_ecn(sk)) 321 INET_ECN_xmit(sk); 322 } 323 324 /* Packet ECN state for a SYN. */ 325 static void tcp_ecn_send_syn(struct sock *sk, struct sk_buff *skb) 326 { 327 struct tcp_sock *tp = tcp_sk(sk); 328 bool bpf_needs_ecn = tcp_bpf_ca_needs_ecn(sk); 329 bool use_ecn = sock_net(sk)->ipv4.sysctl_tcp_ecn == 1 || 330 tcp_ca_needs_ecn(sk) || bpf_needs_ecn; 331 332 if (!use_ecn) { 333 const struct dst_entry *dst = __sk_dst_get(sk); 334 335 if (dst && dst_feature(dst, RTAX_FEATURE_ECN)) 336 use_ecn = true; 337 } 338 339 tp->ecn_flags = 0; 340 341 if (use_ecn) { 342 TCP_SKB_CB(skb)->tcp_flags |= TCPHDR_ECE | TCPHDR_CWR; 343 tp->ecn_flags = TCP_ECN_OK; 344 if (tcp_ca_needs_ecn(sk) || bpf_needs_ecn) 345 INET_ECN_xmit(sk); 346 } 347 } 348 349 static void tcp_ecn_clear_syn(struct sock *sk, struct sk_buff *skb) 350 { 351 if (sock_net(sk)->ipv4.sysctl_tcp_ecn_fallback) 352 /* tp->ecn_flags are cleared at a later point in time when 353 * SYN ACK is ultimatively being received. 354 */ 355 TCP_SKB_CB(skb)->tcp_flags &= ~(TCPHDR_ECE | TCPHDR_CWR); 356 } 357 358 static void 359 tcp_ecn_make_synack(const struct request_sock *req, struct tcphdr *th) 360 { 361 if (inet_rsk(req)->ecn_ok) 362 th->ece = 1; 363 } 364 365 /* Set up ECN state for a packet on a ESTABLISHED socket that is about to 366 * be sent. 367 */ 368 static void tcp_ecn_send(struct sock *sk, struct sk_buff *skb, 369 struct tcphdr *th, int tcp_header_len) 370 { 371 struct tcp_sock *tp = tcp_sk(sk); 372 373 if (tp->ecn_flags & TCP_ECN_OK) { 374 /* Not-retransmitted data segment: set ECT and inject CWR. */ 375 if (skb->len != tcp_header_len && 376 !before(TCP_SKB_CB(skb)->seq, tp->snd_nxt)) { 377 INET_ECN_xmit(sk); 378 if (tp->ecn_flags & TCP_ECN_QUEUE_CWR) { 379 tp->ecn_flags &= ~TCP_ECN_QUEUE_CWR; 380 th->cwr = 1; 381 skb_shinfo(skb)->gso_type |= SKB_GSO_TCP_ECN; 382 } 383 } else if (!tcp_ca_needs_ecn(sk)) { 384 /* ACK or retransmitted segment: clear ECT|CE */ 385 INET_ECN_dontxmit(sk); 386 } 387 if (tp->ecn_flags & TCP_ECN_DEMAND_CWR) 388 th->ece = 1; 389 } 390 } 391 392 /* Constructs common control bits of non-data skb. If SYN/FIN is present, 393 * auto increment end seqno. 394 */ 395 static void tcp_init_nondata_skb(struct sk_buff *skb, u32 seq, u8 flags) 396 { 397 skb->ip_summed = CHECKSUM_PARTIAL; 398 skb->csum = 0; 399 400 TCP_SKB_CB(skb)->tcp_flags = flags; 401 TCP_SKB_CB(skb)->sacked = 0; 402 403 tcp_skb_pcount_set(skb, 1); 404 405 TCP_SKB_CB(skb)->seq = seq; 406 if (flags & (TCPHDR_SYN | TCPHDR_FIN)) 407 seq++; 408 TCP_SKB_CB(skb)->end_seq = seq; 409 } 410 411 static inline bool tcp_urg_mode(const struct tcp_sock *tp) 412 { 413 return tp->snd_una != tp->snd_up; 414 } 415 416 #define OPTION_SACK_ADVERTISE (1 << 0) 417 #define OPTION_TS (1 << 1) 418 #define OPTION_MD5 (1 << 2) 419 #define OPTION_WSCALE (1 << 3) 420 #define OPTION_FAST_OPEN_COOKIE (1 << 8) 421 422 struct tcp_out_options { 423 u16 options; /* bit field of OPTION_* */ 424 u16 mss; /* 0 to disable */ 425 u8 ws; /* window scale, 0 to disable */ 426 u8 num_sack_blocks; /* number of SACK blocks to include */ 427 u8 hash_size; /* bytes in hash_location */ 428 __u8 *hash_location; /* temporary pointer, overloaded */ 429 __u32 tsval, tsecr; /* need to include OPTION_TS */ 430 struct tcp_fastopen_cookie *fastopen_cookie; /* Fast open cookie */ 431 }; 432 433 /* Write previously computed TCP options to the packet. 434 * 435 * Beware: Something in the Internet is very sensitive to the ordering of 436 * TCP options, we learned this through the hard way, so be careful here. 437 * Luckily we can at least blame others for their non-compliance but from 438 * inter-operability perspective it seems that we're somewhat stuck with 439 * the ordering which we have been using if we want to keep working with 440 * those broken things (not that it currently hurts anybody as there isn't 441 * particular reason why the ordering would need to be changed). 442 * 443 * At least SACK_PERM as the first option is known to lead to a disaster 444 * (but it may well be that other scenarios fail similarly). 445 */ 446 static void tcp_options_write(__be32 *ptr, struct tcp_sock *tp, 447 struct tcp_out_options *opts) 448 { 449 u16 options = opts->options; /* mungable copy */ 450 451 if (unlikely(OPTION_MD5 & options)) { 452 *ptr++ = htonl((TCPOPT_NOP << 24) | (TCPOPT_NOP << 16) | 453 (TCPOPT_MD5SIG << 8) | TCPOLEN_MD5SIG); 454 /* overload cookie hash location */ 455 opts->hash_location = (__u8 *)ptr; 456 ptr += 4; 457 } 458 459 if (unlikely(opts->mss)) { 460 *ptr++ = htonl((TCPOPT_MSS << 24) | 461 (TCPOLEN_MSS << 16) | 462 opts->mss); 463 } 464 465 if (likely(OPTION_TS & options)) { 466 if (unlikely(OPTION_SACK_ADVERTISE & options)) { 467 *ptr++ = htonl((TCPOPT_SACK_PERM << 24) | 468 (TCPOLEN_SACK_PERM << 16) | 469 (TCPOPT_TIMESTAMP << 8) | 470 TCPOLEN_TIMESTAMP); 471 options &= ~OPTION_SACK_ADVERTISE; 472 } else { 473 *ptr++ = htonl((TCPOPT_NOP << 24) | 474 (TCPOPT_NOP << 16) | 475 (TCPOPT_TIMESTAMP << 8) | 476 TCPOLEN_TIMESTAMP); 477 } 478 *ptr++ = htonl(opts->tsval); 479 *ptr++ = htonl(opts->tsecr); 480 } 481 482 if (unlikely(OPTION_SACK_ADVERTISE & options)) { 483 *ptr++ = htonl((TCPOPT_NOP << 24) | 484 (TCPOPT_NOP << 16) | 485 (TCPOPT_SACK_PERM << 8) | 486 TCPOLEN_SACK_PERM); 487 } 488 489 if (unlikely(OPTION_WSCALE & options)) { 490 *ptr++ = htonl((TCPOPT_NOP << 24) | 491 (TCPOPT_WINDOW << 16) | 492 (TCPOLEN_WINDOW << 8) | 493 opts->ws); 494 } 495 496 if (unlikely(opts->num_sack_blocks)) { 497 struct tcp_sack_block *sp = tp->rx_opt.dsack ? 498 tp->duplicate_sack : tp->selective_acks; 499 int this_sack; 500 501 *ptr++ = htonl((TCPOPT_NOP << 24) | 502 (TCPOPT_NOP << 16) | 503 (TCPOPT_SACK << 8) | 504 (TCPOLEN_SACK_BASE + (opts->num_sack_blocks * 505 TCPOLEN_SACK_PERBLOCK))); 506 507 for (this_sack = 0; this_sack < opts->num_sack_blocks; 508 ++this_sack) { 509 *ptr++ = htonl(sp[this_sack].start_seq); 510 *ptr++ = htonl(sp[this_sack].end_seq); 511 } 512 513 tp->rx_opt.dsack = 0; 514 } 515 516 if (unlikely(OPTION_FAST_OPEN_COOKIE & options)) { 517 struct tcp_fastopen_cookie *foc = opts->fastopen_cookie; 518 u8 *p = (u8 *)ptr; 519 u32 len; /* Fast Open option length */ 520 521 if (foc->exp) { 522 len = TCPOLEN_EXP_FASTOPEN_BASE + foc->len; 523 *ptr = htonl((TCPOPT_EXP << 24) | (len << 16) | 524 TCPOPT_FASTOPEN_MAGIC); 525 p += TCPOLEN_EXP_FASTOPEN_BASE; 526 } else { 527 len = TCPOLEN_FASTOPEN_BASE + foc->len; 528 *p++ = TCPOPT_FASTOPEN; 529 *p++ = len; 530 } 531 532 memcpy(p, foc->val, foc->len); 533 if ((len & 3) == 2) { 534 p[foc->len] = TCPOPT_NOP; 535 p[foc->len + 1] = TCPOPT_NOP; 536 } 537 ptr += (len + 3) >> 2; 538 } 539 } 540 541 /* Compute TCP options for SYN packets. This is not the final 542 * network wire format yet. 543 */ 544 static unsigned int tcp_syn_options(struct sock *sk, struct sk_buff *skb, 545 struct tcp_out_options *opts, 546 struct tcp_md5sig_key **md5) 547 { 548 struct tcp_sock *tp = tcp_sk(sk); 549 unsigned int remaining = MAX_TCP_OPTION_SPACE; 550 struct tcp_fastopen_request *fastopen = tp->fastopen_req; 551 552 #ifdef CONFIG_TCP_MD5SIG 553 *md5 = tp->af_specific->md5_lookup(sk, sk); 554 if (*md5) { 555 opts->options |= OPTION_MD5; 556 remaining -= TCPOLEN_MD5SIG_ALIGNED; 557 } 558 #else 559 *md5 = NULL; 560 #endif 561 562 /* We always get an MSS option. The option bytes which will be seen in 563 * normal data packets should timestamps be used, must be in the MSS 564 * advertised. But we subtract them from tp->mss_cache so that 565 * calculations in tcp_sendmsg are simpler etc. So account for this 566 * fact here if necessary. If we don't do this correctly, as a 567 * receiver we won't recognize data packets as being full sized when we 568 * should, and thus we won't abide by the delayed ACK rules correctly. 569 * SACKs don't matter, we never delay an ACK when we have any of those 570 * going out. */ 571 opts->mss = tcp_advertise_mss(sk); 572 remaining -= TCPOLEN_MSS_ALIGNED; 573 574 if (likely(sock_net(sk)->ipv4.sysctl_tcp_timestamps && !*md5)) { 575 opts->options |= OPTION_TS; 576 opts->tsval = tcp_skb_timestamp(skb) + tp->tsoffset; 577 opts->tsecr = tp->rx_opt.ts_recent; 578 remaining -= TCPOLEN_TSTAMP_ALIGNED; 579 } 580 if (likely(sock_net(sk)->ipv4.sysctl_tcp_window_scaling)) { 581 opts->ws = tp->rx_opt.rcv_wscale; 582 opts->options |= OPTION_WSCALE; 583 remaining -= TCPOLEN_WSCALE_ALIGNED; 584 } 585 if (likely(sock_net(sk)->ipv4.sysctl_tcp_sack)) { 586 opts->options |= OPTION_SACK_ADVERTISE; 587 if (unlikely(!(OPTION_TS & opts->options))) 588 remaining -= TCPOLEN_SACKPERM_ALIGNED; 589 } 590 591 if (fastopen && fastopen->cookie.len >= 0) { 592 u32 need = fastopen->cookie.len; 593 594 need += fastopen->cookie.exp ? TCPOLEN_EXP_FASTOPEN_BASE : 595 TCPOLEN_FASTOPEN_BASE; 596 need = (need + 3) & ~3U; /* Align to 32 bits */ 597 if (remaining >= need) { 598 opts->options |= OPTION_FAST_OPEN_COOKIE; 599 opts->fastopen_cookie = &fastopen->cookie; 600 remaining -= need; 601 tp->syn_fastopen = 1; 602 tp->syn_fastopen_exp = fastopen->cookie.exp ? 1 : 0; 603 } 604 } 605 606 return MAX_TCP_OPTION_SPACE - remaining; 607 } 608 609 /* Set up TCP options for SYN-ACKs. */ 610 static unsigned int tcp_synack_options(struct request_sock *req, 611 unsigned int mss, struct sk_buff *skb, 612 struct tcp_out_options *opts, 613 const struct tcp_md5sig_key *md5, 614 struct tcp_fastopen_cookie *foc) 615 { 616 struct inet_request_sock *ireq = inet_rsk(req); 617 unsigned int remaining = MAX_TCP_OPTION_SPACE; 618 619 #ifdef CONFIG_TCP_MD5SIG 620 if (md5) { 621 opts->options |= OPTION_MD5; 622 remaining -= TCPOLEN_MD5SIG_ALIGNED; 623 624 /* We can't fit any SACK blocks in a packet with MD5 + TS 625 * options. There was discussion about disabling SACK 626 * rather than TS in order to fit in better with old, 627 * buggy kernels, but that was deemed to be unnecessary. 628 */ 629 ireq->tstamp_ok &= !ireq->sack_ok; 630 } 631 #endif 632 633 /* We always send an MSS option. */ 634 opts->mss = mss; 635 remaining -= TCPOLEN_MSS_ALIGNED; 636 637 if (likely(ireq->wscale_ok)) { 638 opts->ws = ireq->rcv_wscale; 639 opts->options |= OPTION_WSCALE; 640 remaining -= TCPOLEN_WSCALE_ALIGNED; 641 } 642 if (likely(ireq->tstamp_ok)) { 643 opts->options |= OPTION_TS; 644 opts->tsval = tcp_skb_timestamp(skb) + tcp_rsk(req)->ts_off; 645 opts->tsecr = req->ts_recent; 646 remaining -= TCPOLEN_TSTAMP_ALIGNED; 647 } 648 if (likely(ireq->sack_ok)) { 649 opts->options |= OPTION_SACK_ADVERTISE; 650 if (unlikely(!ireq->tstamp_ok)) 651 remaining -= TCPOLEN_SACKPERM_ALIGNED; 652 } 653 if (foc != NULL && foc->len >= 0) { 654 u32 need = foc->len; 655 656 need += foc->exp ? TCPOLEN_EXP_FASTOPEN_BASE : 657 TCPOLEN_FASTOPEN_BASE; 658 need = (need + 3) & ~3U; /* Align to 32 bits */ 659 if (remaining >= need) { 660 opts->options |= OPTION_FAST_OPEN_COOKIE; 661 opts->fastopen_cookie = foc; 662 remaining -= need; 663 } 664 } 665 666 return MAX_TCP_OPTION_SPACE - remaining; 667 } 668 669 /* Compute TCP options for ESTABLISHED sockets. This is not the 670 * final wire format yet. 671 */ 672 static unsigned int tcp_established_options(struct sock *sk, struct sk_buff *skb, 673 struct tcp_out_options *opts, 674 struct tcp_md5sig_key **md5) 675 { 676 struct tcp_sock *tp = tcp_sk(sk); 677 unsigned int size = 0; 678 unsigned int eff_sacks; 679 680 opts->options = 0; 681 682 #ifdef CONFIG_TCP_MD5SIG 683 *md5 = tp->af_specific->md5_lookup(sk, sk); 684 if (unlikely(*md5)) { 685 opts->options |= OPTION_MD5; 686 size += TCPOLEN_MD5SIG_ALIGNED; 687 } 688 #else 689 *md5 = NULL; 690 #endif 691 692 if (likely(tp->rx_opt.tstamp_ok)) { 693 opts->options |= OPTION_TS; 694 opts->tsval = skb ? tcp_skb_timestamp(skb) + tp->tsoffset : 0; 695 opts->tsecr = tp->rx_opt.ts_recent; 696 size += TCPOLEN_TSTAMP_ALIGNED; 697 } 698 699 eff_sacks = tp->rx_opt.num_sacks + tp->rx_opt.dsack; 700 if (unlikely(eff_sacks)) { 701 const unsigned int remaining = MAX_TCP_OPTION_SPACE - size; 702 opts->num_sack_blocks = 703 min_t(unsigned int, eff_sacks, 704 (remaining - TCPOLEN_SACK_BASE_ALIGNED) / 705 TCPOLEN_SACK_PERBLOCK); 706 size += TCPOLEN_SACK_BASE_ALIGNED + 707 opts->num_sack_blocks * TCPOLEN_SACK_PERBLOCK; 708 } 709 710 return size; 711 } 712 713 714 /* TCP SMALL QUEUES (TSQ) 715 * 716 * TSQ goal is to keep small amount of skbs per tcp flow in tx queues (qdisc+dev) 717 * to reduce RTT and bufferbloat. 718 * We do this using a special skb destructor (tcp_wfree). 719 * 720 * Its important tcp_wfree() can be replaced by sock_wfree() in the event skb 721 * needs to be reallocated in a driver. 722 * The invariant being skb->truesize subtracted from sk->sk_wmem_alloc 723 * 724 * Since transmit from skb destructor is forbidden, we use a tasklet 725 * to process all sockets that eventually need to send more skbs. 726 * We use one tasklet per cpu, with its own queue of sockets. 727 */ 728 struct tsq_tasklet { 729 struct tasklet_struct tasklet; 730 struct list_head head; /* queue of tcp sockets */ 731 }; 732 static DEFINE_PER_CPU(struct tsq_tasklet, tsq_tasklet); 733 734 static void tcp_tsq_handler(struct sock *sk) 735 { 736 if ((1 << sk->sk_state) & 737 (TCPF_ESTABLISHED | TCPF_FIN_WAIT1 | TCPF_CLOSING | 738 TCPF_CLOSE_WAIT | TCPF_LAST_ACK)) { 739 struct tcp_sock *tp = tcp_sk(sk); 740 741 if (tp->lost_out > tp->retrans_out && 742 tp->snd_cwnd > tcp_packets_in_flight(tp)) 743 tcp_xmit_retransmit_queue(sk); 744 745 tcp_write_xmit(sk, tcp_current_mss(sk), tp->nonagle, 746 0, GFP_ATOMIC); 747 } 748 } 749 /* 750 * One tasklet per cpu tries to send more skbs. 751 * We run in tasklet context but need to disable irqs when 752 * transferring tsq->head because tcp_wfree() might 753 * interrupt us (non NAPI drivers) 754 */ 755 static void tcp_tasklet_func(unsigned long data) 756 { 757 struct tsq_tasklet *tsq = (struct tsq_tasklet *)data; 758 LIST_HEAD(list); 759 unsigned long flags; 760 struct list_head *q, *n; 761 struct tcp_sock *tp; 762 struct sock *sk; 763 764 local_irq_save(flags); 765 list_splice_init(&tsq->head, &list); 766 local_irq_restore(flags); 767 768 list_for_each_safe(q, n, &list) { 769 tp = list_entry(q, struct tcp_sock, tsq_node); 770 list_del(&tp->tsq_node); 771 772 sk = (struct sock *)tp; 773 smp_mb__before_atomic(); 774 clear_bit(TSQ_QUEUED, &sk->sk_tsq_flags); 775 776 if (!sk->sk_lock.owned && 777 test_bit(TCP_TSQ_DEFERRED, &sk->sk_tsq_flags)) { 778 bh_lock_sock(sk); 779 if (!sock_owned_by_user(sk)) { 780 clear_bit(TCP_TSQ_DEFERRED, &sk->sk_tsq_flags); 781 tcp_tsq_handler(sk); 782 } 783 bh_unlock_sock(sk); 784 } 785 786 sk_free(sk); 787 } 788 } 789 790 #define TCP_DEFERRED_ALL (TCPF_TSQ_DEFERRED | \ 791 TCPF_WRITE_TIMER_DEFERRED | \ 792 TCPF_DELACK_TIMER_DEFERRED | \ 793 TCPF_MTU_REDUCED_DEFERRED) 794 /** 795 * tcp_release_cb - tcp release_sock() callback 796 * @sk: socket 797 * 798 * called from release_sock() to perform protocol dependent 799 * actions before socket release. 800 */ 801 void tcp_release_cb(struct sock *sk) 802 { 803 unsigned long flags, nflags; 804 805 /* perform an atomic operation only if at least one flag is set */ 806 do { 807 flags = sk->sk_tsq_flags; 808 if (!(flags & TCP_DEFERRED_ALL)) 809 return; 810 nflags = flags & ~TCP_DEFERRED_ALL; 811 } while (cmpxchg(&sk->sk_tsq_flags, flags, nflags) != flags); 812 813 if (flags & TCPF_TSQ_DEFERRED) 814 tcp_tsq_handler(sk); 815 816 /* Here begins the tricky part : 817 * We are called from release_sock() with : 818 * 1) BH disabled 819 * 2) sk_lock.slock spinlock held 820 * 3) socket owned by us (sk->sk_lock.owned == 1) 821 * 822 * But following code is meant to be called from BH handlers, 823 * so we should keep BH disabled, but early release socket ownership 824 */ 825 sock_release_ownership(sk); 826 827 if (flags & TCPF_WRITE_TIMER_DEFERRED) { 828 tcp_write_timer_handler(sk); 829 __sock_put(sk); 830 } 831 if (flags & TCPF_DELACK_TIMER_DEFERRED) { 832 tcp_delack_timer_handler(sk); 833 __sock_put(sk); 834 } 835 if (flags & TCPF_MTU_REDUCED_DEFERRED) { 836 inet_csk(sk)->icsk_af_ops->mtu_reduced(sk); 837 __sock_put(sk); 838 } 839 } 840 EXPORT_SYMBOL(tcp_release_cb); 841 842 void __init tcp_tasklet_init(void) 843 { 844 int i; 845 846 for_each_possible_cpu(i) { 847 struct tsq_tasklet *tsq = &per_cpu(tsq_tasklet, i); 848 849 INIT_LIST_HEAD(&tsq->head); 850 tasklet_init(&tsq->tasklet, 851 tcp_tasklet_func, 852 (unsigned long)tsq); 853 } 854 } 855 856 /* 857 * Write buffer destructor automatically called from kfree_skb. 858 * We can't xmit new skbs from this context, as we might already 859 * hold qdisc lock. 860 */ 861 void tcp_wfree(struct sk_buff *skb) 862 { 863 struct sock *sk = skb->sk; 864 struct tcp_sock *tp = tcp_sk(sk); 865 unsigned long flags, nval, oval; 866 867 /* Keep one reference on sk_wmem_alloc. 868 * Will be released by sk_free() from here or tcp_tasklet_func() 869 */ 870 WARN_ON(refcount_sub_and_test(skb->truesize - 1, &sk->sk_wmem_alloc)); 871 872 /* If this softirq is serviced by ksoftirqd, we are likely under stress. 873 * Wait until our queues (qdisc + devices) are drained. 874 * This gives : 875 * - less callbacks to tcp_write_xmit(), reducing stress (batches) 876 * - chance for incoming ACK (processed by another cpu maybe) 877 * to migrate this flow (skb->ooo_okay will be eventually set) 878 */ 879 if (refcount_read(&sk->sk_wmem_alloc) >= SKB_TRUESIZE(1) && this_cpu_ksoftirqd() == current) 880 goto out; 881 882 for (oval = READ_ONCE(sk->sk_tsq_flags);; oval = nval) { 883 struct tsq_tasklet *tsq; 884 bool empty; 885 886 if (!(oval & TSQF_THROTTLED) || (oval & TSQF_QUEUED)) 887 goto out; 888 889 nval = (oval & ~TSQF_THROTTLED) | TSQF_QUEUED | TCPF_TSQ_DEFERRED; 890 nval = cmpxchg(&sk->sk_tsq_flags, oval, nval); 891 if (nval != oval) 892 continue; 893 894 /* queue this socket to tasklet queue */ 895 local_irq_save(flags); 896 tsq = this_cpu_ptr(&tsq_tasklet); 897 empty = list_empty(&tsq->head); 898 list_add(&tp->tsq_node, &tsq->head); 899 if (empty) 900 tasklet_schedule(&tsq->tasklet); 901 local_irq_restore(flags); 902 return; 903 } 904 out: 905 sk_free(sk); 906 } 907 908 /* Note: Called under hard irq. 909 * We can not call TCP stack right away. 910 */ 911 enum hrtimer_restart tcp_pace_kick(struct hrtimer *timer) 912 { 913 struct tcp_sock *tp = container_of(timer, struct tcp_sock, pacing_timer); 914 struct sock *sk = (struct sock *)tp; 915 unsigned long nval, oval; 916 917 for (oval = READ_ONCE(sk->sk_tsq_flags);; oval = nval) { 918 struct tsq_tasklet *tsq; 919 bool empty; 920 921 if (oval & TSQF_QUEUED) 922 break; 923 924 nval = (oval & ~TSQF_THROTTLED) | TSQF_QUEUED | TCPF_TSQ_DEFERRED; 925 nval = cmpxchg(&sk->sk_tsq_flags, oval, nval); 926 if (nval != oval) 927 continue; 928 929 if (!refcount_inc_not_zero(&sk->sk_wmem_alloc)) 930 break; 931 /* queue this socket to tasklet queue */ 932 tsq = this_cpu_ptr(&tsq_tasklet); 933 empty = list_empty(&tsq->head); 934 list_add(&tp->tsq_node, &tsq->head); 935 if (empty) 936 tasklet_schedule(&tsq->tasklet); 937 break; 938 } 939 return HRTIMER_NORESTART; 940 } 941 942 /* BBR congestion control needs pacing. 943 * Same remark for SO_MAX_PACING_RATE. 944 * sch_fq packet scheduler is efficiently handling pacing, 945 * but is not always installed/used. 946 * Return true if TCP stack should pace packets itself. 947 */ 948 static bool tcp_needs_internal_pacing(const struct sock *sk) 949 { 950 return smp_load_acquire(&sk->sk_pacing_status) == SK_PACING_NEEDED; 951 } 952 953 static void tcp_internal_pacing(struct sock *sk, const struct sk_buff *skb) 954 { 955 u64 len_ns; 956 u32 rate; 957 958 if (!tcp_needs_internal_pacing(sk)) 959 return; 960 rate = sk->sk_pacing_rate; 961 if (!rate || rate == ~0U) 962 return; 963 964 /* Should account for header sizes as sch_fq does, 965 * but lets make things simple. 966 */ 967 len_ns = (u64)skb->len * NSEC_PER_SEC; 968 do_div(len_ns, rate); 969 hrtimer_start(&tcp_sk(sk)->pacing_timer, 970 ktime_add_ns(ktime_get(), len_ns), 971 HRTIMER_MODE_ABS_PINNED); 972 } 973 974 /* This routine actually transmits TCP packets queued in by 975 * tcp_do_sendmsg(). This is used by both the initial 976 * transmission and possible later retransmissions. 977 * All SKB's seen here are completely headerless. It is our 978 * job to build the TCP header, and pass the packet down to 979 * IP so it can do the same plus pass the packet off to the 980 * device. 981 * 982 * We are working here with either a clone of the original 983 * SKB, or a fresh unique copy made by the retransmit engine. 984 */ 985 static int tcp_transmit_skb(struct sock *sk, struct sk_buff *skb, int clone_it, 986 gfp_t gfp_mask) 987 { 988 const struct inet_connection_sock *icsk = inet_csk(sk); 989 struct inet_sock *inet; 990 struct tcp_sock *tp; 991 struct tcp_skb_cb *tcb; 992 struct tcp_out_options opts; 993 unsigned int tcp_options_size, tcp_header_size; 994 struct sk_buff *oskb = NULL; 995 struct tcp_md5sig_key *md5; 996 struct tcphdr *th; 997 int err; 998 999 BUG_ON(!skb || !tcp_skb_pcount(skb)); 1000 tp = tcp_sk(sk); 1001 1002 if (clone_it) { 1003 TCP_SKB_CB(skb)->tx.in_flight = TCP_SKB_CB(skb)->end_seq 1004 - tp->snd_una; 1005 oskb = skb; 1006 if (unlikely(skb_cloned(skb))) 1007 skb = pskb_copy(skb, gfp_mask); 1008 else 1009 skb = skb_clone(skb, gfp_mask); 1010 if (unlikely(!skb)) 1011 return -ENOBUFS; 1012 } 1013 skb->skb_mstamp = tp->tcp_mstamp; 1014 1015 inet = inet_sk(sk); 1016 tcb = TCP_SKB_CB(skb); 1017 memset(&opts, 0, sizeof(opts)); 1018 1019 if (unlikely(tcb->tcp_flags & TCPHDR_SYN)) 1020 tcp_options_size = tcp_syn_options(sk, skb, &opts, &md5); 1021 else 1022 tcp_options_size = tcp_established_options(sk, skb, &opts, 1023 &md5); 1024 tcp_header_size = tcp_options_size + sizeof(struct tcphdr); 1025 1026 /* if no packet is in qdisc/device queue, then allow XPS to select 1027 * another queue. We can be called from tcp_tsq_handler() 1028 * which holds one reference to sk_wmem_alloc. 1029 * 1030 * TODO: Ideally, in-flight pure ACK packets should not matter here. 1031 * One way to get this would be to set skb->truesize = 2 on them. 1032 */ 1033 skb->ooo_okay = sk_wmem_alloc_get(sk) < SKB_TRUESIZE(1); 1034 1035 /* If we had to use memory reserve to allocate this skb, 1036 * this might cause drops if packet is looped back : 1037 * Other socket might not have SOCK_MEMALLOC. 1038 * Packets not looped back do not care about pfmemalloc. 1039 */ 1040 skb->pfmemalloc = 0; 1041 1042 skb_push(skb, tcp_header_size); 1043 skb_reset_transport_header(skb); 1044 1045 skb_orphan(skb); 1046 skb->sk = sk; 1047 skb->destructor = skb_is_tcp_pure_ack(skb) ? __sock_wfree : tcp_wfree; 1048 skb_set_hash_from_sk(skb, sk); 1049 refcount_add(skb->truesize, &sk->sk_wmem_alloc); 1050 1051 skb_set_dst_pending_confirm(skb, sk->sk_dst_pending_confirm); 1052 1053 /* Build TCP header and checksum it. */ 1054 th = (struct tcphdr *)skb->data; 1055 th->source = inet->inet_sport; 1056 th->dest = inet->inet_dport; 1057 th->seq = htonl(tcb->seq); 1058 th->ack_seq = htonl(tp->rcv_nxt); 1059 *(((__be16 *)th) + 6) = htons(((tcp_header_size >> 2) << 12) | 1060 tcb->tcp_flags); 1061 1062 th->check = 0; 1063 th->urg_ptr = 0; 1064 1065 /* The urg_mode check is necessary during a below snd_una win probe */ 1066 if (unlikely(tcp_urg_mode(tp) && before(tcb->seq, tp->snd_up))) { 1067 if (before(tp->snd_up, tcb->seq + 0x10000)) { 1068 th->urg_ptr = htons(tp->snd_up - tcb->seq); 1069 th->urg = 1; 1070 } else if (after(tcb->seq + 0xFFFF, tp->snd_nxt)) { 1071 th->urg_ptr = htons(0xFFFF); 1072 th->urg = 1; 1073 } 1074 } 1075 1076 tcp_options_write((__be32 *)(th + 1), tp, &opts); 1077 skb_shinfo(skb)->gso_type = sk->sk_gso_type; 1078 if (likely(!(tcb->tcp_flags & TCPHDR_SYN))) { 1079 th->window = htons(tcp_select_window(sk)); 1080 tcp_ecn_send(sk, skb, th, tcp_header_size); 1081 } else { 1082 /* RFC1323: The window in SYN & SYN/ACK segments 1083 * is never scaled. 1084 */ 1085 th->window = htons(min(tp->rcv_wnd, 65535U)); 1086 } 1087 #ifdef CONFIG_TCP_MD5SIG 1088 /* Calculate the MD5 hash, as we have all we need now */ 1089 if (md5) { 1090 sk_nocaps_add(sk, NETIF_F_GSO_MASK); 1091 tp->af_specific->calc_md5_hash(opts.hash_location, 1092 md5, sk, skb); 1093 } 1094 #endif 1095 1096 icsk->icsk_af_ops->send_check(sk, skb); 1097 1098 if (likely(tcb->tcp_flags & TCPHDR_ACK)) 1099 tcp_event_ack_sent(sk, tcp_skb_pcount(skb)); 1100 1101 if (skb->len != tcp_header_size) { 1102 tcp_event_data_sent(tp, sk); 1103 tp->data_segs_out += tcp_skb_pcount(skb); 1104 tcp_internal_pacing(sk, skb); 1105 } 1106 1107 if (after(tcb->end_seq, tp->snd_nxt) || tcb->seq == tcb->end_seq) 1108 TCP_ADD_STATS(sock_net(sk), TCP_MIB_OUTSEGS, 1109 tcp_skb_pcount(skb)); 1110 1111 tp->segs_out += tcp_skb_pcount(skb); 1112 /* OK, its time to fill skb_shinfo(skb)->gso_{segs|size} */ 1113 skb_shinfo(skb)->gso_segs = tcp_skb_pcount(skb); 1114 skb_shinfo(skb)->gso_size = tcp_skb_mss(skb); 1115 1116 /* Our usage of tstamp should remain private */ 1117 skb->tstamp = 0; 1118 1119 /* Cleanup our debris for IP stacks */ 1120 memset(skb->cb, 0, max(sizeof(struct inet_skb_parm), 1121 sizeof(struct inet6_skb_parm))); 1122 1123 err = icsk->icsk_af_ops->queue_xmit(sk, skb, &inet->cork.fl); 1124 1125 if (unlikely(err > 0)) { 1126 tcp_enter_cwr(sk); 1127 err = net_xmit_eval(err); 1128 } 1129 if (!err && oskb) { 1130 oskb->skb_mstamp = tp->tcp_mstamp; 1131 tcp_rate_skb_sent(sk, oskb); 1132 } 1133 return err; 1134 } 1135 1136 /* This routine just queues the buffer for sending. 1137 * 1138 * NOTE: probe0 timer is not checked, do not forget tcp_push_pending_frames, 1139 * otherwise socket can stall. 1140 */ 1141 static void tcp_queue_skb(struct sock *sk, struct sk_buff *skb) 1142 { 1143 struct tcp_sock *tp = tcp_sk(sk); 1144 1145 /* Advance write_seq and place onto the write_queue. */ 1146 tp->write_seq = TCP_SKB_CB(skb)->end_seq; 1147 __skb_header_release(skb); 1148 tcp_add_write_queue_tail(sk, skb); 1149 sk->sk_wmem_queued += skb->truesize; 1150 sk_mem_charge(sk, skb->truesize); 1151 } 1152 1153 /* Initialize TSO segments for a packet. */ 1154 static void tcp_set_skb_tso_segs(struct sk_buff *skb, unsigned int mss_now) 1155 { 1156 if (skb->len <= mss_now || skb->ip_summed == CHECKSUM_NONE) { 1157 /* Avoid the costly divide in the normal 1158 * non-TSO case. 1159 */ 1160 tcp_skb_pcount_set(skb, 1); 1161 TCP_SKB_CB(skb)->tcp_gso_size = 0; 1162 } else { 1163 tcp_skb_pcount_set(skb, DIV_ROUND_UP(skb->len, mss_now)); 1164 TCP_SKB_CB(skb)->tcp_gso_size = mss_now; 1165 } 1166 } 1167 1168 /* When a modification to fackets out becomes necessary, we need to check 1169 * skb is counted to fackets_out or not. 1170 */ 1171 static void tcp_adjust_fackets_out(struct sock *sk, const struct sk_buff *skb, 1172 int decr) 1173 { 1174 struct tcp_sock *tp = tcp_sk(sk); 1175 1176 if (!tp->sacked_out || tcp_is_reno(tp)) 1177 return; 1178 1179 if (after(tcp_highest_sack_seq(tp), TCP_SKB_CB(skb)->seq)) 1180 tp->fackets_out -= decr; 1181 } 1182 1183 /* Pcount in the middle of the write queue got changed, we need to do various 1184 * tweaks to fix counters 1185 */ 1186 static void tcp_adjust_pcount(struct sock *sk, const struct sk_buff *skb, int decr) 1187 { 1188 struct tcp_sock *tp = tcp_sk(sk); 1189 1190 tp->packets_out -= decr; 1191 1192 if (TCP_SKB_CB(skb)->sacked & TCPCB_SACKED_ACKED) 1193 tp->sacked_out -= decr; 1194 if (TCP_SKB_CB(skb)->sacked & TCPCB_SACKED_RETRANS) 1195 tp->retrans_out -= decr; 1196 if (TCP_SKB_CB(skb)->sacked & TCPCB_LOST) 1197 tp->lost_out -= decr; 1198 1199 /* Reno case is special. Sigh... */ 1200 if (tcp_is_reno(tp) && decr > 0) 1201 tp->sacked_out -= min_t(u32, tp->sacked_out, decr); 1202 1203 tcp_adjust_fackets_out(sk, skb, decr); 1204 1205 if (tp->lost_skb_hint && 1206 before(TCP_SKB_CB(skb)->seq, TCP_SKB_CB(tp->lost_skb_hint)->seq) && 1207 (tcp_is_fack(tp) || (TCP_SKB_CB(skb)->sacked & TCPCB_SACKED_ACKED))) 1208 tp->lost_cnt_hint -= decr; 1209 1210 tcp_verify_left_out(tp); 1211 } 1212 1213 static bool tcp_has_tx_tstamp(const struct sk_buff *skb) 1214 { 1215 return TCP_SKB_CB(skb)->txstamp_ack || 1216 (skb_shinfo(skb)->tx_flags & SKBTX_ANY_TSTAMP); 1217 } 1218 1219 static void tcp_fragment_tstamp(struct sk_buff *skb, struct sk_buff *skb2) 1220 { 1221 struct skb_shared_info *shinfo = skb_shinfo(skb); 1222 1223 if (unlikely(tcp_has_tx_tstamp(skb)) && 1224 !before(shinfo->tskey, TCP_SKB_CB(skb2)->seq)) { 1225 struct skb_shared_info *shinfo2 = skb_shinfo(skb2); 1226 u8 tsflags = shinfo->tx_flags & SKBTX_ANY_TSTAMP; 1227 1228 shinfo->tx_flags &= ~tsflags; 1229 shinfo2->tx_flags |= tsflags; 1230 swap(shinfo->tskey, shinfo2->tskey); 1231 TCP_SKB_CB(skb2)->txstamp_ack = TCP_SKB_CB(skb)->txstamp_ack; 1232 TCP_SKB_CB(skb)->txstamp_ack = 0; 1233 } 1234 } 1235 1236 static void tcp_skb_fragment_eor(struct sk_buff *skb, struct sk_buff *skb2) 1237 { 1238 TCP_SKB_CB(skb2)->eor = TCP_SKB_CB(skb)->eor; 1239 TCP_SKB_CB(skb)->eor = 0; 1240 } 1241 1242 /* Function to create two new TCP segments. Shrinks the given segment 1243 * to the specified size and appends a new segment with the rest of the 1244 * packet to the list. This won't be called frequently, I hope. 1245 * Remember, these are still headerless SKBs at this point. 1246 */ 1247 int tcp_fragment(struct sock *sk, struct sk_buff *skb, u32 len, 1248 unsigned int mss_now, gfp_t gfp) 1249 { 1250 struct tcp_sock *tp = tcp_sk(sk); 1251 struct sk_buff *buff; 1252 int nsize, old_factor; 1253 int nlen; 1254 u8 flags; 1255 1256 if (WARN_ON(len > skb->len)) 1257 return -EINVAL; 1258 1259 nsize = skb_headlen(skb) - len; 1260 if (nsize < 0) 1261 nsize = 0; 1262 1263 if (skb_unclone(skb, gfp)) 1264 return -ENOMEM; 1265 1266 /* Get a new skb... force flag on. */ 1267 buff = sk_stream_alloc_skb(sk, nsize, gfp, true); 1268 if (!buff) 1269 return -ENOMEM; /* We'll just try again later. */ 1270 1271 sk->sk_wmem_queued += buff->truesize; 1272 sk_mem_charge(sk, buff->truesize); 1273 nlen = skb->len - len - nsize; 1274 buff->truesize += nlen; 1275 skb->truesize -= nlen; 1276 1277 /* Correct the sequence numbers. */ 1278 TCP_SKB_CB(buff)->seq = TCP_SKB_CB(skb)->seq + len; 1279 TCP_SKB_CB(buff)->end_seq = TCP_SKB_CB(skb)->end_seq; 1280 TCP_SKB_CB(skb)->end_seq = TCP_SKB_CB(buff)->seq; 1281 1282 /* PSH and FIN should only be set in the second packet. */ 1283 flags = TCP_SKB_CB(skb)->tcp_flags; 1284 TCP_SKB_CB(skb)->tcp_flags = flags & ~(TCPHDR_FIN | TCPHDR_PSH); 1285 TCP_SKB_CB(buff)->tcp_flags = flags; 1286 TCP_SKB_CB(buff)->sacked = TCP_SKB_CB(skb)->sacked; 1287 tcp_skb_fragment_eor(skb, buff); 1288 1289 if (!skb_shinfo(skb)->nr_frags && skb->ip_summed != CHECKSUM_PARTIAL) { 1290 /* Copy and checksum data tail into the new buffer. */ 1291 buff->csum = csum_partial_copy_nocheck(skb->data + len, 1292 skb_put(buff, nsize), 1293 nsize, 0); 1294 1295 skb_trim(skb, len); 1296 1297 skb->csum = csum_block_sub(skb->csum, buff->csum, len); 1298 } else { 1299 skb->ip_summed = CHECKSUM_PARTIAL; 1300 skb_split(skb, buff, len); 1301 } 1302 1303 buff->ip_summed = skb->ip_summed; 1304 1305 buff->tstamp = skb->tstamp; 1306 tcp_fragment_tstamp(skb, buff); 1307 1308 old_factor = tcp_skb_pcount(skb); 1309 1310 /* Fix up tso_factor for both original and new SKB. */ 1311 tcp_set_skb_tso_segs(skb, mss_now); 1312 tcp_set_skb_tso_segs(buff, mss_now); 1313 1314 /* Update delivered info for the new segment */ 1315 TCP_SKB_CB(buff)->tx = TCP_SKB_CB(skb)->tx; 1316 1317 /* If this packet has been sent out already, we must 1318 * adjust the various packet counters. 1319 */ 1320 if (!before(tp->snd_nxt, TCP_SKB_CB(buff)->end_seq)) { 1321 int diff = old_factor - tcp_skb_pcount(skb) - 1322 tcp_skb_pcount(buff); 1323 1324 if (diff) 1325 tcp_adjust_pcount(sk, skb, diff); 1326 } 1327 1328 /* Link BUFF into the send queue. */ 1329 __skb_header_release(buff); 1330 tcp_insert_write_queue_after(skb, buff, sk); 1331 1332 return 0; 1333 } 1334 1335 /* This is similar to __pskb_pull_tail(). The difference is that pulled 1336 * data is not copied, but immediately discarded. 1337 */ 1338 static int __pskb_trim_head(struct sk_buff *skb, int len) 1339 { 1340 struct skb_shared_info *shinfo; 1341 int i, k, eat; 1342 1343 eat = min_t(int, len, skb_headlen(skb)); 1344 if (eat) { 1345 __skb_pull(skb, eat); 1346 len -= eat; 1347 if (!len) 1348 return 0; 1349 } 1350 eat = len; 1351 k = 0; 1352 shinfo = skb_shinfo(skb); 1353 for (i = 0; i < shinfo->nr_frags; i++) { 1354 int size = skb_frag_size(&shinfo->frags[i]); 1355 1356 if (size <= eat) { 1357 skb_frag_unref(skb, i); 1358 eat -= size; 1359 } else { 1360 shinfo->frags[k] = shinfo->frags[i]; 1361 if (eat) { 1362 shinfo->frags[k].page_offset += eat; 1363 skb_frag_size_sub(&shinfo->frags[k], eat); 1364 eat = 0; 1365 } 1366 k++; 1367 } 1368 } 1369 shinfo->nr_frags = k; 1370 1371 skb->data_len -= len; 1372 skb->len = skb->data_len; 1373 return len; 1374 } 1375 1376 /* Remove acked data from a packet in the transmit queue. */ 1377 int tcp_trim_head(struct sock *sk, struct sk_buff *skb, u32 len) 1378 { 1379 u32 delta_truesize; 1380 1381 if (skb_unclone(skb, GFP_ATOMIC)) 1382 return -ENOMEM; 1383 1384 delta_truesize = __pskb_trim_head(skb, len); 1385 1386 TCP_SKB_CB(skb)->seq += len; 1387 skb->ip_summed = CHECKSUM_PARTIAL; 1388 1389 if (delta_truesize) { 1390 skb->truesize -= delta_truesize; 1391 sk->sk_wmem_queued -= delta_truesize; 1392 sk_mem_uncharge(sk, delta_truesize); 1393 sock_set_flag(sk, SOCK_QUEUE_SHRUNK); 1394 } 1395 1396 /* Any change of skb->len requires recalculation of tso factor. */ 1397 if (tcp_skb_pcount(skb) > 1) 1398 tcp_set_skb_tso_segs(skb, tcp_skb_mss(skb)); 1399 1400 return 0; 1401 } 1402 1403 /* Calculate MSS not accounting any TCP options. */ 1404 static inline int __tcp_mtu_to_mss(struct sock *sk, int pmtu) 1405 { 1406 const struct tcp_sock *tp = tcp_sk(sk); 1407 const struct inet_connection_sock *icsk = inet_csk(sk); 1408 int mss_now; 1409 1410 /* Calculate base mss without TCP options: 1411 It is MMS_S - sizeof(tcphdr) of rfc1122 1412 */ 1413 mss_now = pmtu - icsk->icsk_af_ops->net_header_len - sizeof(struct tcphdr); 1414 1415 /* IPv6 adds a frag_hdr in case RTAX_FEATURE_ALLFRAG is set */ 1416 if (icsk->icsk_af_ops->net_frag_header_len) { 1417 const struct dst_entry *dst = __sk_dst_get(sk); 1418 1419 if (dst && dst_allfrag(dst)) 1420 mss_now -= icsk->icsk_af_ops->net_frag_header_len; 1421 } 1422 1423 /* Clamp it (mss_clamp does not include tcp options) */ 1424 if (mss_now > tp->rx_opt.mss_clamp) 1425 mss_now = tp->rx_opt.mss_clamp; 1426 1427 /* Now subtract optional transport overhead */ 1428 mss_now -= icsk->icsk_ext_hdr_len; 1429 1430 /* Then reserve room for full set of TCP options and 8 bytes of data */ 1431 if (mss_now < 48) 1432 mss_now = 48; 1433 return mss_now; 1434 } 1435 1436 /* Calculate MSS. Not accounting for SACKs here. */ 1437 int tcp_mtu_to_mss(struct sock *sk, int pmtu) 1438 { 1439 /* Subtract TCP options size, not including SACKs */ 1440 return __tcp_mtu_to_mss(sk, pmtu) - 1441 (tcp_sk(sk)->tcp_header_len - sizeof(struct tcphdr)); 1442 } 1443 1444 /* Inverse of above */ 1445 int tcp_mss_to_mtu(struct sock *sk, int mss) 1446 { 1447 const struct tcp_sock *tp = tcp_sk(sk); 1448 const struct inet_connection_sock *icsk = inet_csk(sk); 1449 int mtu; 1450 1451 mtu = mss + 1452 tp->tcp_header_len + 1453 icsk->icsk_ext_hdr_len + 1454 icsk->icsk_af_ops->net_header_len; 1455 1456 /* IPv6 adds a frag_hdr in case RTAX_FEATURE_ALLFRAG is set */ 1457 if (icsk->icsk_af_ops->net_frag_header_len) { 1458 const struct dst_entry *dst = __sk_dst_get(sk); 1459 1460 if (dst && dst_allfrag(dst)) 1461 mtu += icsk->icsk_af_ops->net_frag_header_len; 1462 } 1463 return mtu; 1464 } 1465 EXPORT_SYMBOL(tcp_mss_to_mtu); 1466 1467 /* MTU probing init per socket */ 1468 void tcp_mtup_init(struct sock *sk) 1469 { 1470 struct tcp_sock *tp = tcp_sk(sk); 1471 struct inet_connection_sock *icsk = inet_csk(sk); 1472 struct net *net = sock_net(sk); 1473 1474 icsk->icsk_mtup.enabled = net->ipv4.sysctl_tcp_mtu_probing > 1; 1475 icsk->icsk_mtup.search_high = tp->rx_opt.mss_clamp + sizeof(struct tcphdr) + 1476 icsk->icsk_af_ops->net_header_len; 1477 icsk->icsk_mtup.search_low = tcp_mss_to_mtu(sk, net->ipv4.sysctl_tcp_base_mss); 1478 icsk->icsk_mtup.probe_size = 0; 1479 if (icsk->icsk_mtup.enabled) 1480 icsk->icsk_mtup.probe_timestamp = tcp_jiffies32; 1481 } 1482 EXPORT_SYMBOL(tcp_mtup_init); 1483 1484 /* This function synchronize snd mss to current pmtu/exthdr set. 1485 1486 tp->rx_opt.user_mss is mss set by user by TCP_MAXSEG. It does NOT counts 1487 for TCP options, but includes only bare TCP header. 1488 1489 tp->rx_opt.mss_clamp is mss negotiated at connection setup. 1490 It is minimum of user_mss and mss received with SYN. 1491 It also does not include TCP options. 1492 1493 inet_csk(sk)->icsk_pmtu_cookie is last pmtu, seen by this function. 1494 1495 tp->mss_cache is current effective sending mss, including 1496 all tcp options except for SACKs. It is evaluated, 1497 taking into account current pmtu, but never exceeds 1498 tp->rx_opt.mss_clamp. 1499 1500 NOTE1. rfc1122 clearly states that advertised MSS 1501 DOES NOT include either tcp or ip options. 1502 1503 NOTE2. inet_csk(sk)->icsk_pmtu_cookie and tp->mss_cache 1504 are READ ONLY outside this function. --ANK (980731) 1505 */ 1506 unsigned int tcp_sync_mss(struct sock *sk, u32 pmtu) 1507 { 1508 struct tcp_sock *tp = tcp_sk(sk); 1509 struct inet_connection_sock *icsk = inet_csk(sk); 1510 int mss_now; 1511 1512 if (icsk->icsk_mtup.search_high > pmtu) 1513 icsk->icsk_mtup.search_high = pmtu; 1514 1515 mss_now = tcp_mtu_to_mss(sk, pmtu); 1516 mss_now = tcp_bound_to_half_wnd(tp, mss_now); 1517 1518 /* And store cached results */ 1519 icsk->icsk_pmtu_cookie = pmtu; 1520 if (icsk->icsk_mtup.enabled) 1521 mss_now = min(mss_now, tcp_mtu_to_mss(sk, icsk->icsk_mtup.search_low)); 1522 tp->mss_cache = mss_now; 1523 1524 return mss_now; 1525 } 1526 EXPORT_SYMBOL(tcp_sync_mss); 1527 1528 /* Compute the current effective MSS, taking SACKs and IP options, 1529 * and even PMTU discovery events into account. 1530 */ 1531 unsigned int tcp_current_mss(struct sock *sk) 1532 { 1533 const struct tcp_sock *tp = tcp_sk(sk); 1534 const struct dst_entry *dst = __sk_dst_get(sk); 1535 u32 mss_now; 1536 unsigned int header_len; 1537 struct tcp_out_options opts; 1538 struct tcp_md5sig_key *md5; 1539 1540 mss_now = tp->mss_cache; 1541 1542 if (dst) { 1543 u32 mtu = dst_mtu(dst); 1544 if (mtu != inet_csk(sk)->icsk_pmtu_cookie) 1545 mss_now = tcp_sync_mss(sk, mtu); 1546 } 1547 1548 header_len = tcp_established_options(sk, NULL, &opts, &md5) + 1549 sizeof(struct tcphdr); 1550 /* The mss_cache is sized based on tp->tcp_header_len, which assumes 1551 * some common options. If this is an odd packet (because we have SACK 1552 * blocks etc) then our calculated header_len will be different, and 1553 * we have to adjust mss_now correspondingly */ 1554 if (header_len != tp->tcp_header_len) { 1555 int delta = (int) header_len - tp->tcp_header_len; 1556 mss_now -= delta; 1557 } 1558 1559 return mss_now; 1560 } 1561 1562 /* RFC2861, slow part. Adjust cwnd, after it was not full during one rto. 1563 * As additional protections, we do not touch cwnd in retransmission phases, 1564 * and if application hit its sndbuf limit recently. 1565 */ 1566 static void tcp_cwnd_application_limited(struct sock *sk) 1567 { 1568 struct tcp_sock *tp = tcp_sk(sk); 1569 1570 if (inet_csk(sk)->icsk_ca_state == TCP_CA_Open && 1571 sk->sk_socket && !test_bit(SOCK_NOSPACE, &sk->sk_socket->flags)) { 1572 /* Limited by application or receiver window. */ 1573 u32 init_win = tcp_init_cwnd(tp, __sk_dst_get(sk)); 1574 u32 win_used = max(tp->snd_cwnd_used, init_win); 1575 if (win_used < tp->snd_cwnd) { 1576 tp->snd_ssthresh = tcp_current_ssthresh(sk); 1577 tp->snd_cwnd = (tp->snd_cwnd + win_used) >> 1; 1578 } 1579 tp->snd_cwnd_used = 0; 1580 } 1581 tp->snd_cwnd_stamp = tcp_jiffies32; 1582 } 1583 1584 static void tcp_cwnd_validate(struct sock *sk, bool is_cwnd_limited) 1585 { 1586 const struct tcp_congestion_ops *ca_ops = inet_csk(sk)->icsk_ca_ops; 1587 struct tcp_sock *tp = tcp_sk(sk); 1588 1589 /* Track the maximum number of outstanding packets in each 1590 * window, and remember whether we were cwnd-limited then. 1591 */ 1592 if (!before(tp->snd_una, tp->max_packets_seq) || 1593 tp->packets_out > tp->max_packets_out) { 1594 tp->max_packets_out = tp->packets_out; 1595 tp->max_packets_seq = tp->snd_nxt; 1596 tp->is_cwnd_limited = is_cwnd_limited; 1597 } 1598 1599 if (tcp_is_cwnd_limited(sk)) { 1600 /* Network is feed fully. */ 1601 tp->snd_cwnd_used = 0; 1602 tp->snd_cwnd_stamp = tcp_jiffies32; 1603 } else { 1604 /* Network starves. */ 1605 if (tp->packets_out > tp->snd_cwnd_used) 1606 tp->snd_cwnd_used = tp->packets_out; 1607 1608 if (sysctl_tcp_slow_start_after_idle && 1609 (s32)(tcp_jiffies32 - tp->snd_cwnd_stamp) >= inet_csk(sk)->icsk_rto && 1610 !ca_ops->cong_control) 1611 tcp_cwnd_application_limited(sk); 1612 1613 /* The following conditions together indicate the starvation 1614 * is caused by insufficient sender buffer: 1615 * 1) just sent some data (see tcp_write_xmit) 1616 * 2) not cwnd limited (this else condition) 1617 * 3) no more data to send (null tcp_send_head ) 1618 * 4) application is hitting buffer limit (SOCK_NOSPACE) 1619 */ 1620 if (!tcp_send_head(sk) && sk->sk_socket && 1621 test_bit(SOCK_NOSPACE, &sk->sk_socket->flags) && 1622 (1 << sk->sk_state) & (TCPF_ESTABLISHED | TCPF_CLOSE_WAIT)) 1623 tcp_chrono_start(sk, TCP_CHRONO_SNDBUF_LIMITED); 1624 } 1625 } 1626 1627 /* Minshall's variant of the Nagle send check. */ 1628 static bool tcp_minshall_check(const struct tcp_sock *tp) 1629 { 1630 return after(tp->snd_sml, tp->snd_una) && 1631 !after(tp->snd_sml, tp->snd_nxt); 1632 } 1633 1634 /* Update snd_sml if this skb is under mss 1635 * Note that a TSO packet might end with a sub-mss segment 1636 * The test is really : 1637 * if ((skb->len % mss) != 0) 1638 * tp->snd_sml = TCP_SKB_CB(skb)->end_seq; 1639 * But we can avoid doing the divide again given we already have 1640 * skb_pcount = skb->len / mss_now 1641 */ 1642 static void tcp_minshall_update(struct tcp_sock *tp, unsigned int mss_now, 1643 const struct sk_buff *skb) 1644 { 1645 if (skb->len < tcp_skb_pcount(skb) * mss_now) 1646 tp->snd_sml = TCP_SKB_CB(skb)->end_seq; 1647 } 1648 1649 /* Return false, if packet can be sent now without violation Nagle's rules: 1650 * 1. It is full sized. (provided by caller in %partial bool) 1651 * 2. Or it contains FIN. (already checked by caller) 1652 * 3. Or TCP_CORK is not set, and TCP_NODELAY is set. 1653 * 4. Or TCP_CORK is not set, and all sent packets are ACKed. 1654 * With Minshall's modification: all sent small packets are ACKed. 1655 */ 1656 static bool tcp_nagle_check(bool partial, const struct tcp_sock *tp, 1657 int nonagle) 1658 { 1659 return partial && 1660 ((nonagle & TCP_NAGLE_CORK) || 1661 (!nonagle && tp->packets_out && tcp_minshall_check(tp))); 1662 } 1663 1664 /* Return how many segs we'd like on a TSO packet, 1665 * to send one TSO packet per ms 1666 */ 1667 u32 tcp_tso_autosize(const struct sock *sk, unsigned int mss_now, 1668 int min_tso_segs) 1669 { 1670 u32 bytes, segs; 1671 1672 bytes = min(sk->sk_pacing_rate >> 10, 1673 sk->sk_gso_max_size - 1 - MAX_TCP_HEADER); 1674 1675 /* Goal is to send at least one packet per ms, 1676 * not one big TSO packet every 100 ms. 1677 * This preserves ACK clocking and is consistent 1678 * with tcp_tso_should_defer() heuristic. 1679 */ 1680 segs = max_t(u32, bytes / mss_now, min_tso_segs); 1681 1682 return min_t(u32, segs, sk->sk_gso_max_segs); 1683 } 1684 EXPORT_SYMBOL(tcp_tso_autosize); 1685 1686 /* Return the number of segments we want in the skb we are transmitting. 1687 * See if congestion control module wants to decide; otherwise, autosize. 1688 */ 1689 static u32 tcp_tso_segs(struct sock *sk, unsigned int mss_now) 1690 { 1691 const struct tcp_congestion_ops *ca_ops = inet_csk(sk)->icsk_ca_ops; 1692 u32 tso_segs = ca_ops->tso_segs_goal ? ca_ops->tso_segs_goal(sk) : 0; 1693 1694 return tso_segs ? : 1695 tcp_tso_autosize(sk, mss_now, sysctl_tcp_min_tso_segs); 1696 } 1697 1698 /* Returns the portion of skb which can be sent right away */ 1699 static unsigned int tcp_mss_split_point(const struct sock *sk, 1700 const struct sk_buff *skb, 1701 unsigned int mss_now, 1702 unsigned int max_segs, 1703 int nonagle) 1704 { 1705 const struct tcp_sock *tp = tcp_sk(sk); 1706 u32 partial, needed, window, max_len; 1707 1708 window = tcp_wnd_end(tp) - TCP_SKB_CB(skb)->seq; 1709 max_len = mss_now * max_segs; 1710 1711 if (likely(max_len <= window && skb != tcp_write_queue_tail(sk))) 1712 return max_len; 1713 1714 needed = min(skb->len, window); 1715 1716 if (max_len <= needed) 1717 return max_len; 1718 1719 partial = needed % mss_now; 1720 /* If last segment is not a full MSS, check if Nagle rules allow us 1721 * to include this last segment in this skb. 1722 * Otherwise, we'll split the skb at last MSS boundary 1723 */ 1724 if (tcp_nagle_check(partial != 0, tp, nonagle)) 1725 return needed - partial; 1726 1727 return needed; 1728 } 1729 1730 /* Can at least one segment of SKB be sent right now, according to the 1731 * congestion window rules? If so, return how many segments are allowed. 1732 */ 1733 static inline unsigned int tcp_cwnd_test(const struct tcp_sock *tp, 1734 const struct sk_buff *skb) 1735 { 1736 u32 in_flight, cwnd, halfcwnd; 1737 1738 /* Don't be strict about the congestion window for the final FIN. */ 1739 if ((TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN) && 1740 tcp_skb_pcount(skb) == 1) 1741 return 1; 1742 1743 in_flight = tcp_packets_in_flight(tp); 1744 cwnd = tp->snd_cwnd; 1745 if (in_flight >= cwnd) 1746 return 0; 1747 1748 /* For better scheduling, ensure we have at least 1749 * 2 GSO packets in flight. 1750 */ 1751 halfcwnd = max(cwnd >> 1, 1U); 1752 return min(halfcwnd, cwnd - in_flight); 1753 } 1754 1755 /* Initialize TSO state of a skb. 1756 * This must be invoked the first time we consider transmitting 1757 * SKB onto the wire. 1758 */ 1759 static int tcp_init_tso_segs(struct sk_buff *skb, unsigned int mss_now) 1760 { 1761 int tso_segs = tcp_skb_pcount(skb); 1762 1763 if (!tso_segs || (tso_segs > 1 && tcp_skb_mss(skb) != mss_now)) { 1764 tcp_set_skb_tso_segs(skb, mss_now); 1765 tso_segs = tcp_skb_pcount(skb); 1766 } 1767 return tso_segs; 1768 } 1769 1770 1771 /* Return true if the Nagle test allows this packet to be 1772 * sent now. 1773 */ 1774 static inline bool tcp_nagle_test(const struct tcp_sock *tp, const struct sk_buff *skb, 1775 unsigned int cur_mss, int nonagle) 1776 { 1777 /* Nagle rule does not apply to frames, which sit in the middle of the 1778 * write_queue (they have no chances to get new data). 1779 * 1780 * This is implemented in the callers, where they modify the 'nonagle' 1781 * argument based upon the location of SKB in the send queue. 1782 */ 1783 if (nonagle & TCP_NAGLE_PUSH) 1784 return true; 1785 1786 /* Don't use the nagle rule for urgent data (or for the final FIN). */ 1787 if (tcp_urg_mode(tp) || (TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN)) 1788 return true; 1789 1790 if (!tcp_nagle_check(skb->len < cur_mss, tp, nonagle)) 1791 return true; 1792 1793 return false; 1794 } 1795 1796 /* Does at least the first segment of SKB fit into the send window? */ 1797 static bool tcp_snd_wnd_test(const struct tcp_sock *tp, 1798 const struct sk_buff *skb, 1799 unsigned int cur_mss) 1800 { 1801 u32 end_seq = TCP_SKB_CB(skb)->end_seq; 1802 1803 if (skb->len > cur_mss) 1804 end_seq = TCP_SKB_CB(skb)->seq + cur_mss; 1805 1806 return !after(end_seq, tcp_wnd_end(tp)); 1807 } 1808 1809 /* Trim TSO SKB to LEN bytes, put the remaining data into a new packet 1810 * which is put after SKB on the list. It is very much like 1811 * tcp_fragment() except that it may make several kinds of assumptions 1812 * in order to speed up the splitting operation. In particular, we 1813 * know that all the data is in scatter-gather pages, and that the 1814 * packet has never been sent out before (and thus is not cloned). 1815 */ 1816 static int tso_fragment(struct sock *sk, struct sk_buff *skb, unsigned int len, 1817 unsigned int mss_now, gfp_t gfp) 1818 { 1819 struct sk_buff *buff; 1820 int nlen = skb->len - len; 1821 u8 flags; 1822 1823 /* All of a TSO frame must be composed of paged data. */ 1824 if (skb->len != skb->data_len) 1825 return tcp_fragment(sk, skb, len, mss_now, gfp); 1826 1827 buff = sk_stream_alloc_skb(sk, 0, gfp, true); 1828 if (unlikely(!buff)) 1829 return -ENOMEM; 1830 1831 sk->sk_wmem_queued += buff->truesize; 1832 sk_mem_charge(sk, buff->truesize); 1833 buff->truesize += nlen; 1834 skb->truesize -= nlen; 1835 1836 /* Correct the sequence numbers. */ 1837 TCP_SKB_CB(buff)->seq = TCP_SKB_CB(skb)->seq + len; 1838 TCP_SKB_CB(buff)->end_seq = TCP_SKB_CB(skb)->end_seq; 1839 TCP_SKB_CB(skb)->end_seq = TCP_SKB_CB(buff)->seq; 1840 1841 /* PSH and FIN should only be set in the second packet. */ 1842 flags = TCP_SKB_CB(skb)->tcp_flags; 1843 TCP_SKB_CB(skb)->tcp_flags = flags & ~(TCPHDR_FIN | TCPHDR_PSH); 1844 TCP_SKB_CB(buff)->tcp_flags = flags; 1845 1846 /* This packet was never sent out yet, so no SACK bits. */ 1847 TCP_SKB_CB(buff)->sacked = 0; 1848 1849 tcp_skb_fragment_eor(skb, buff); 1850 1851 buff->ip_summed = skb->ip_summed = CHECKSUM_PARTIAL; 1852 skb_split(skb, buff, len); 1853 tcp_fragment_tstamp(skb, buff); 1854 1855 /* Fix up tso_factor for both original and new SKB. */ 1856 tcp_set_skb_tso_segs(skb, mss_now); 1857 tcp_set_skb_tso_segs(buff, mss_now); 1858 1859 /* Link BUFF into the send queue. */ 1860 __skb_header_release(buff); 1861 tcp_insert_write_queue_after(skb, buff, sk); 1862 1863 return 0; 1864 } 1865 1866 /* Try to defer sending, if possible, in order to minimize the amount 1867 * of TSO splitting we do. View it as a kind of TSO Nagle test. 1868 * 1869 * This algorithm is from John Heffner. 1870 */ 1871 static bool tcp_tso_should_defer(struct sock *sk, struct sk_buff *skb, 1872 bool *is_cwnd_limited, u32 max_segs) 1873 { 1874 const struct inet_connection_sock *icsk = inet_csk(sk); 1875 u32 age, send_win, cong_win, limit, in_flight; 1876 struct tcp_sock *tp = tcp_sk(sk); 1877 struct sk_buff *head; 1878 int win_divisor; 1879 1880 if (TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN) 1881 goto send_now; 1882 1883 if (icsk->icsk_ca_state >= TCP_CA_Recovery) 1884 goto send_now; 1885 1886 /* Avoid bursty behavior by allowing defer 1887 * only if the last write was recent. 1888 */ 1889 if ((s32)(tcp_jiffies32 - tp->lsndtime) > 0) 1890 goto send_now; 1891 1892 in_flight = tcp_packets_in_flight(tp); 1893 1894 BUG_ON(tcp_skb_pcount(skb) <= 1 || (tp->snd_cwnd <= in_flight)); 1895 1896 send_win = tcp_wnd_end(tp) - TCP_SKB_CB(skb)->seq; 1897 1898 /* From in_flight test above, we know that cwnd > in_flight. */ 1899 cong_win = (tp->snd_cwnd - in_flight) * tp->mss_cache; 1900 1901 limit = min(send_win, cong_win); 1902 1903 /* If a full-sized TSO skb can be sent, do it. */ 1904 if (limit >= max_segs * tp->mss_cache) 1905 goto send_now; 1906 1907 /* Middle in queue won't get any more data, full sendable already? */ 1908 if ((skb != tcp_write_queue_tail(sk)) && (limit >= skb->len)) 1909 goto send_now; 1910 1911 win_divisor = ACCESS_ONCE(sysctl_tcp_tso_win_divisor); 1912 if (win_divisor) { 1913 u32 chunk = min(tp->snd_wnd, tp->snd_cwnd * tp->mss_cache); 1914 1915 /* If at least some fraction of a window is available, 1916 * just use it. 1917 */ 1918 chunk /= win_divisor; 1919 if (limit >= chunk) 1920 goto send_now; 1921 } else { 1922 /* Different approach, try not to defer past a single 1923 * ACK. Receiver should ACK every other full sized 1924 * frame, so if we have space for more than 3 frames 1925 * then send now. 1926 */ 1927 if (limit > tcp_max_tso_deferred_mss(tp) * tp->mss_cache) 1928 goto send_now; 1929 } 1930 1931 head = tcp_write_queue_head(sk); 1932 1933 age = tcp_stamp_us_delta(tp->tcp_mstamp, head->skb_mstamp); 1934 /* If next ACK is likely to come too late (half srtt), do not defer */ 1935 if (age < (tp->srtt_us >> 4)) 1936 goto send_now; 1937 1938 /* Ok, it looks like it is advisable to defer. */ 1939 1940 if (cong_win < send_win && cong_win <= skb->len) 1941 *is_cwnd_limited = true; 1942 1943 return true; 1944 1945 send_now: 1946 return false; 1947 } 1948 1949 static inline void tcp_mtu_check_reprobe(struct sock *sk) 1950 { 1951 struct inet_connection_sock *icsk = inet_csk(sk); 1952 struct tcp_sock *tp = tcp_sk(sk); 1953 struct net *net = sock_net(sk); 1954 u32 interval; 1955 s32 delta; 1956 1957 interval = net->ipv4.sysctl_tcp_probe_interval; 1958 delta = tcp_jiffies32 - icsk->icsk_mtup.probe_timestamp; 1959 if (unlikely(delta >= interval * HZ)) { 1960 int mss = tcp_current_mss(sk); 1961 1962 /* Update current search range */ 1963 icsk->icsk_mtup.probe_size = 0; 1964 icsk->icsk_mtup.search_high = tp->rx_opt.mss_clamp + 1965 sizeof(struct tcphdr) + 1966 icsk->icsk_af_ops->net_header_len; 1967 icsk->icsk_mtup.search_low = tcp_mss_to_mtu(sk, mss); 1968 1969 /* Update probe time stamp */ 1970 icsk->icsk_mtup.probe_timestamp = tcp_jiffies32; 1971 } 1972 } 1973 1974 /* Create a new MTU probe if we are ready. 1975 * MTU probe is regularly attempting to increase the path MTU by 1976 * deliberately sending larger packets. This discovers routing 1977 * changes resulting in larger path MTUs. 1978 * 1979 * Returns 0 if we should wait to probe (no cwnd available), 1980 * 1 if a probe was sent, 1981 * -1 otherwise 1982 */ 1983 static int tcp_mtu_probe(struct sock *sk) 1984 { 1985 struct inet_connection_sock *icsk = inet_csk(sk); 1986 struct tcp_sock *tp = tcp_sk(sk); 1987 struct sk_buff *skb, *nskb, *next; 1988 struct net *net = sock_net(sk); 1989 int probe_size; 1990 int size_needed; 1991 int copy, len; 1992 int mss_now; 1993 int interval; 1994 1995 /* Not currently probing/verifying, 1996 * not in recovery, 1997 * have enough cwnd, and 1998 * not SACKing (the variable headers throw things off) 1999 */ 2000 if (likely(!icsk->icsk_mtup.enabled || 2001 icsk->icsk_mtup.probe_size || 2002 inet_csk(sk)->icsk_ca_state != TCP_CA_Open || 2003 tp->snd_cwnd < 11 || 2004 tp->rx_opt.num_sacks || tp->rx_opt.dsack)) 2005 return -1; 2006 2007 /* Use binary search for probe_size between tcp_mss_base, 2008 * and current mss_clamp. if (search_high - search_low) 2009 * smaller than a threshold, backoff from probing. 2010 */ 2011 mss_now = tcp_current_mss(sk); 2012 probe_size = tcp_mtu_to_mss(sk, (icsk->icsk_mtup.search_high + 2013 icsk->icsk_mtup.search_low) >> 1); 2014 size_needed = probe_size + (tp->reordering + 1) * tp->mss_cache; 2015 interval = icsk->icsk_mtup.search_high - icsk->icsk_mtup.search_low; 2016 /* When misfortune happens, we are reprobing actively, 2017 * and then reprobe timer has expired. We stick with current 2018 * probing process by not resetting search range to its orignal. 2019 */ 2020 if (probe_size > tcp_mtu_to_mss(sk, icsk->icsk_mtup.search_high) || 2021 interval < net->ipv4.sysctl_tcp_probe_threshold) { 2022 /* Check whether enough time has elaplased for 2023 * another round of probing. 2024 */ 2025 tcp_mtu_check_reprobe(sk); 2026 return -1; 2027 } 2028 2029 /* Have enough data in the send queue to probe? */ 2030 if (tp->write_seq - tp->snd_nxt < size_needed) 2031 return -1; 2032 2033 if (tp->snd_wnd < size_needed) 2034 return -1; 2035 if (after(tp->snd_nxt + size_needed, tcp_wnd_end(tp))) 2036 return 0; 2037 2038 /* Do we need to wait to drain cwnd? With none in flight, don't stall */ 2039 if (tcp_packets_in_flight(tp) + 2 > tp->snd_cwnd) { 2040 if (!tcp_packets_in_flight(tp)) 2041 return -1; 2042 else 2043 return 0; 2044 } 2045 2046 /* We're allowed to probe. Build it now. */ 2047 nskb = sk_stream_alloc_skb(sk, probe_size, GFP_ATOMIC, false); 2048 if (!nskb) 2049 return -1; 2050 sk->sk_wmem_queued += nskb->truesize; 2051 sk_mem_charge(sk, nskb->truesize); 2052 2053 skb = tcp_send_head(sk); 2054 2055 TCP_SKB_CB(nskb)->seq = TCP_SKB_CB(skb)->seq; 2056 TCP_SKB_CB(nskb)->end_seq = TCP_SKB_CB(skb)->seq + probe_size; 2057 TCP_SKB_CB(nskb)->tcp_flags = TCPHDR_ACK; 2058 TCP_SKB_CB(nskb)->sacked = 0; 2059 nskb->csum = 0; 2060 nskb->ip_summed = skb->ip_summed; 2061 2062 tcp_insert_write_queue_before(nskb, skb, sk); 2063 2064 len = 0; 2065 tcp_for_write_queue_from_safe(skb, next, sk) { 2066 copy = min_t(int, skb->len, probe_size - len); 2067 if (nskb->ip_summed) { 2068 skb_copy_bits(skb, 0, skb_put(nskb, copy), copy); 2069 } else { 2070 __wsum csum = skb_copy_and_csum_bits(skb, 0, 2071 skb_put(nskb, copy), 2072 copy, 0); 2073 nskb->csum = csum_block_add(nskb->csum, csum, len); 2074 } 2075 2076 if (skb->len <= copy) { 2077 /* We've eaten all the data from this skb. 2078 * Throw it away. */ 2079 TCP_SKB_CB(nskb)->tcp_flags |= TCP_SKB_CB(skb)->tcp_flags; 2080 tcp_unlink_write_queue(skb, sk); 2081 sk_wmem_free_skb(sk, skb); 2082 } else { 2083 TCP_SKB_CB(nskb)->tcp_flags |= TCP_SKB_CB(skb)->tcp_flags & 2084 ~(TCPHDR_FIN|TCPHDR_PSH); 2085 if (!skb_shinfo(skb)->nr_frags) { 2086 skb_pull(skb, copy); 2087 if (skb->ip_summed != CHECKSUM_PARTIAL) 2088 skb->csum = csum_partial(skb->data, 2089 skb->len, 0); 2090 } else { 2091 __pskb_trim_head(skb, copy); 2092 tcp_set_skb_tso_segs(skb, mss_now); 2093 } 2094 TCP_SKB_CB(skb)->seq += copy; 2095 } 2096 2097 len += copy; 2098 2099 if (len >= probe_size) 2100 break; 2101 } 2102 tcp_init_tso_segs(nskb, nskb->len); 2103 2104 /* We're ready to send. If this fails, the probe will 2105 * be resegmented into mss-sized pieces by tcp_write_xmit(). 2106 */ 2107 if (!tcp_transmit_skb(sk, nskb, 1, GFP_ATOMIC)) { 2108 /* Decrement cwnd here because we are sending 2109 * effectively two packets. */ 2110 tp->snd_cwnd--; 2111 tcp_event_new_data_sent(sk, nskb); 2112 2113 icsk->icsk_mtup.probe_size = tcp_mss_to_mtu(sk, nskb->len); 2114 tp->mtu_probe.probe_seq_start = TCP_SKB_CB(nskb)->seq; 2115 tp->mtu_probe.probe_seq_end = TCP_SKB_CB(nskb)->end_seq; 2116 2117 return 1; 2118 } 2119 2120 return -1; 2121 } 2122 2123 static bool tcp_pacing_check(const struct sock *sk) 2124 { 2125 return tcp_needs_internal_pacing(sk) && 2126 hrtimer_active(&tcp_sk(sk)->pacing_timer); 2127 } 2128 2129 /* TCP Small Queues : 2130 * Control number of packets in qdisc/devices to two packets / or ~1 ms. 2131 * (These limits are doubled for retransmits) 2132 * This allows for : 2133 * - better RTT estimation and ACK scheduling 2134 * - faster recovery 2135 * - high rates 2136 * Alas, some drivers / subsystems require a fair amount 2137 * of queued bytes to ensure line rate. 2138 * One example is wifi aggregation (802.11 AMPDU) 2139 */ 2140 static bool tcp_small_queue_check(struct sock *sk, const struct sk_buff *skb, 2141 unsigned int factor) 2142 { 2143 unsigned int limit; 2144 2145 limit = max(2 * skb->truesize, sk->sk_pacing_rate >> 10); 2146 limit = min_t(u32, limit, sysctl_tcp_limit_output_bytes); 2147 limit <<= factor; 2148 2149 if (refcount_read(&sk->sk_wmem_alloc) > limit) { 2150 /* Always send the 1st or 2nd skb in write queue. 2151 * No need to wait for TX completion to call us back, 2152 * after softirq/tasklet schedule. 2153 * This helps when TX completions are delayed too much. 2154 */ 2155 if (skb == sk->sk_write_queue.next || 2156 skb->prev == sk->sk_write_queue.next) 2157 return false; 2158 2159 set_bit(TSQ_THROTTLED, &sk->sk_tsq_flags); 2160 /* It is possible TX completion already happened 2161 * before we set TSQ_THROTTLED, so we must 2162 * test again the condition. 2163 */ 2164 smp_mb__after_atomic(); 2165 if (refcount_read(&sk->sk_wmem_alloc) > limit) 2166 return true; 2167 } 2168 return false; 2169 } 2170 2171 static void tcp_chrono_set(struct tcp_sock *tp, const enum tcp_chrono new) 2172 { 2173 const u32 now = tcp_jiffies32; 2174 enum tcp_chrono old = tp->chrono_type; 2175 2176 if (old > TCP_CHRONO_UNSPEC) 2177 tp->chrono_stat[old - 1] += now - tp->chrono_start; 2178 tp->chrono_start = now; 2179 tp->chrono_type = new; 2180 } 2181 2182 void tcp_chrono_start(struct sock *sk, const enum tcp_chrono type) 2183 { 2184 struct tcp_sock *tp = tcp_sk(sk); 2185 2186 /* If there are multiple conditions worthy of tracking in a 2187 * chronograph then the highest priority enum takes precedence 2188 * over the other conditions. So that if something "more interesting" 2189 * starts happening, stop the previous chrono and start a new one. 2190 */ 2191 if (type > tp->chrono_type) 2192 tcp_chrono_set(tp, type); 2193 } 2194 2195 void tcp_chrono_stop(struct sock *sk, const enum tcp_chrono type) 2196 { 2197 struct tcp_sock *tp = tcp_sk(sk); 2198 2199 2200 /* There are multiple conditions worthy of tracking in a 2201 * chronograph, so that the highest priority enum takes 2202 * precedence over the other conditions (see tcp_chrono_start). 2203 * If a condition stops, we only stop chrono tracking if 2204 * it's the "most interesting" or current chrono we are 2205 * tracking and starts busy chrono if we have pending data. 2206 */ 2207 if (tcp_write_queue_empty(sk)) 2208 tcp_chrono_set(tp, TCP_CHRONO_UNSPEC); 2209 else if (type == tp->chrono_type) 2210 tcp_chrono_set(tp, TCP_CHRONO_BUSY); 2211 } 2212 2213 /* This routine writes packets to the network. It advances the 2214 * send_head. This happens as incoming acks open up the remote 2215 * window for us. 2216 * 2217 * LARGESEND note: !tcp_urg_mode is overkill, only frames between 2218 * snd_up-64k-mss .. snd_up cannot be large. However, taking into 2219 * account rare use of URG, this is not a big flaw. 2220 * 2221 * Send at most one packet when push_one > 0. Temporarily ignore 2222 * cwnd limit to force at most one packet out when push_one == 2. 2223 2224 * Returns true, if no segments are in flight and we have queued segments, 2225 * but cannot send anything now because of SWS or another problem. 2226 */ 2227 static bool tcp_write_xmit(struct sock *sk, unsigned int mss_now, int nonagle, 2228 int push_one, gfp_t gfp) 2229 { 2230 struct tcp_sock *tp = tcp_sk(sk); 2231 struct sk_buff *skb; 2232 unsigned int tso_segs, sent_pkts; 2233 int cwnd_quota; 2234 int result; 2235 bool is_cwnd_limited = false, is_rwnd_limited = false; 2236 u32 max_segs; 2237 2238 sent_pkts = 0; 2239 2240 if (!push_one) { 2241 /* Do MTU probing. */ 2242 result = tcp_mtu_probe(sk); 2243 if (!result) { 2244 return false; 2245 } else if (result > 0) { 2246 sent_pkts = 1; 2247 } 2248 } 2249 2250 max_segs = tcp_tso_segs(sk, mss_now); 2251 tcp_mstamp_refresh(tp); 2252 while ((skb = tcp_send_head(sk))) { 2253 unsigned int limit; 2254 2255 if (tcp_pacing_check(sk)) 2256 break; 2257 2258 tso_segs = tcp_init_tso_segs(skb, mss_now); 2259 BUG_ON(!tso_segs); 2260 2261 if (unlikely(tp->repair) && tp->repair_queue == TCP_SEND_QUEUE) { 2262 /* "skb_mstamp" is used as a start point for the retransmit timer */ 2263 skb->skb_mstamp = tp->tcp_mstamp; 2264 goto repair; /* Skip network transmission */ 2265 } 2266 2267 cwnd_quota = tcp_cwnd_test(tp, skb); 2268 if (!cwnd_quota) { 2269 if (push_one == 2) 2270 /* Force out a loss probe pkt. */ 2271 cwnd_quota = 1; 2272 else 2273 break; 2274 } 2275 2276 if (unlikely(!tcp_snd_wnd_test(tp, skb, mss_now))) { 2277 is_rwnd_limited = true; 2278 break; 2279 } 2280 2281 if (tso_segs == 1) { 2282 if (unlikely(!tcp_nagle_test(tp, skb, mss_now, 2283 (tcp_skb_is_last(sk, skb) ? 2284 nonagle : TCP_NAGLE_PUSH)))) 2285 break; 2286 } else { 2287 if (!push_one && 2288 tcp_tso_should_defer(sk, skb, &is_cwnd_limited, 2289 max_segs)) 2290 break; 2291 } 2292 2293 limit = mss_now; 2294 if (tso_segs > 1 && !tcp_urg_mode(tp)) 2295 limit = tcp_mss_split_point(sk, skb, mss_now, 2296 min_t(unsigned int, 2297 cwnd_quota, 2298 max_segs), 2299 nonagle); 2300 2301 if (skb->len > limit && 2302 unlikely(tso_fragment(sk, skb, limit, mss_now, gfp))) 2303 break; 2304 2305 if (test_bit(TCP_TSQ_DEFERRED, &sk->sk_tsq_flags)) 2306 clear_bit(TCP_TSQ_DEFERRED, &sk->sk_tsq_flags); 2307 if (tcp_small_queue_check(sk, skb, 0)) 2308 break; 2309 2310 if (unlikely(tcp_transmit_skb(sk, skb, 1, gfp))) 2311 break; 2312 2313 repair: 2314 /* Advance the send_head. This one is sent out. 2315 * This call will increment packets_out. 2316 */ 2317 tcp_event_new_data_sent(sk, skb); 2318 2319 tcp_minshall_update(tp, mss_now, skb); 2320 sent_pkts += tcp_skb_pcount(skb); 2321 2322 if (push_one) 2323 break; 2324 } 2325 2326 if (is_rwnd_limited) 2327 tcp_chrono_start(sk, TCP_CHRONO_RWND_LIMITED); 2328 else 2329 tcp_chrono_stop(sk, TCP_CHRONO_RWND_LIMITED); 2330 2331 if (likely(sent_pkts)) { 2332 if (tcp_in_cwnd_reduction(sk)) 2333 tp->prr_out += sent_pkts; 2334 2335 /* Send one loss probe per tail loss episode. */ 2336 if (push_one != 2) 2337 tcp_schedule_loss_probe(sk); 2338 is_cwnd_limited |= (tcp_packets_in_flight(tp) >= tp->snd_cwnd); 2339 tcp_cwnd_validate(sk, is_cwnd_limited); 2340 return false; 2341 } 2342 return !tp->packets_out && tcp_send_head(sk); 2343 } 2344 2345 bool tcp_schedule_loss_probe(struct sock *sk) 2346 { 2347 struct inet_connection_sock *icsk = inet_csk(sk); 2348 struct tcp_sock *tp = tcp_sk(sk); 2349 u32 timeout, rto_delta_us; 2350 2351 /* Don't do any loss probe on a Fast Open connection before 3WHS 2352 * finishes. 2353 */ 2354 if (tp->fastopen_rsk) 2355 return false; 2356 2357 /* Schedule a loss probe in 2*RTT for SACK capable connections 2358 * in Open state, that are either limited by cwnd or application. 2359 */ 2360 if ((sysctl_tcp_early_retrans != 3 && sysctl_tcp_early_retrans != 4) || 2361 !tp->packets_out || !tcp_is_sack(tp) || 2362 icsk->icsk_ca_state != TCP_CA_Open) 2363 return false; 2364 2365 if ((tp->snd_cwnd > tcp_packets_in_flight(tp)) && 2366 tcp_send_head(sk)) 2367 return false; 2368 2369 /* Probe timeout is 2*rtt. Add minimum RTO to account 2370 * for delayed ack when there's one outstanding packet. If no RTT 2371 * sample is available then probe after TCP_TIMEOUT_INIT. 2372 */ 2373 if (tp->srtt_us) { 2374 timeout = usecs_to_jiffies(tp->srtt_us >> 2); 2375 if (tp->packets_out == 1) 2376 timeout += TCP_RTO_MIN; 2377 else 2378 timeout += TCP_TIMEOUT_MIN; 2379 } else { 2380 timeout = TCP_TIMEOUT_INIT; 2381 } 2382 2383 /* If the RTO formula yields an earlier time, then use that time. */ 2384 rto_delta_us = tcp_rto_delta_us(sk); /* How far in future is RTO? */ 2385 if (rto_delta_us > 0) 2386 timeout = min_t(u32, timeout, usecs_to_jiffies(rto_delta_us)); 2387 2388 inet_csk_reset_xmit_timer(sk, ICSK_TIME_LOSS_PROBE, timeout, 2389 TCP_RTO_MAX); 2390 return true; 2391 } 2392 2393 /* Thanks to skb fast clones, we can detect if a prior transmit of 2394 * a packet is still in a qdisc or driver queue. 2395 * In this case, there is very little point doing a retransmit ! 2396 */ 2397 static bool skb_still_in_host_queue(const struct sock *sk, 2398 const struct sk_buff *skb) 2399 { 2400 if (unlikely(skb_fclone_busy(sk, skb))) { 2401 NET_INC_STATS(sock_net(sk), 2402 LINUX_MIB_TCPSPURIOUS_RTX_HOSTQUEUES); 2403 return true; 2404 } 2405 return false; 2406 } 2407 2408 /* When probe timeout (PTO) fires, try send a new segment if possible, else 2409 * retransmit the last segment. 2410 */ 2411 void tcp_send_loss_probe(struct sock *sk) 2412 { 2413 struct tcp_sock *tp = tcp_sk(sk); 2414 struct sk_buff *skb; 2415 int pcount; 2416 int mss = tcp_current_mss(sk); 2417 2418 skb = tcp_send_head(sk); 2419 if (skb) { 2420 if (tcp_snd_wnd_test(tp, skb, mss)) { 2421 pcount = tp->packets_out; 2422 tcp_write_xmit(sk, mss, TCP_NAGLE_OFF, 2, GFP_ATOMIC); 2423 if (tp->packets_out > pcount) 2424 goto probe_sent; 2425 goto rearm_timer; 2426 } 2427 skb = tcp_write_queue_prev(sk, skb); 2428 } else { 2429 skb = tcp_write_queue_tail(sk); 2430 } 2431 2432 /* At most one outstanding TLP retransmission. */ 2433 if (tp->tlp_high_seq) 2434 goto rearm_timer; 2435 2436 /* Retransmit last segment. */ 2437 if (WARN_ON(!skb)) 2438 goto rearm_timer; 2439 2440 if (skb_still_in_host_queue(sk, skb)) 2441 goto rearm_timer; 2442 2443 pcount = tcp_skb_pcount(skb); 2444 if (WARN_ON(!pcount)) 2445 goto rearm_timer; 2446 2447 if ((pcount > 1) && (skb->len > (pcount - 1) * mss)) { 2448 if (unlikely(tcp_fragment(sk, skb, (pcount - 1) * mss, mss, 2449 GFP_ATOMIC))) 2450 goto rearm_timer; 2451 skb = tcp_write_queue_next(sk, skb); 2452 } 2453 2454 if (WARN_ON(!skb || !tcp_skb_pcount(skb))) 2455 goto rearm_timer; 2456 2457 if (__tcp_retransmit_skb(sk, skb, 1)) 2458 goto rearm_timer; 2459 2460 /* Record snd_nxt for loss detection. */ 2461 tp->tlp_high_seq = tp->snd_nxt; 2462 2463 probe_sent: 2464 NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPLOSSPROBES); 2465 /* Reset s.t. tcp_rearm_rto will restart timer from now */ 2466 inet_csk(sk)->icsk_pending = 0; 2467 rearm_timer: 2468 tcp_rearm_rto(sk); 2469 } 2470 2471 /* Push out any pending frames which were held back due to 2472 * TCP_CORK or attempt at coalescing tiny packets. 2473 * The socket must be locked by the caller. 2474 */ 2475 void __tcp_push_pending_frames(struct sock *sk, unsigned int cur_mss, 2476 int nonagle) 2477 { 2478 /* If we are closed, the bytes will have to remain here. 2479 * In time closedown will finish, we empty the write queue and 2480 * all will be happy. 2481 */ 2482 if (unlikely(sk->sk_state == TCP_CLOSE)) 2483 return; 2484 2485 if (tcp_write_xmit(sk, cur_mss, nonagle, 0, 2486 sk_gfp_mask(sk, GFP_ATOMIC))) 2487 tcp_check_probe_timer(sk); 2488 } 2489 2490 /* Send _single_ skb sitting at the send head. This function requires 2491 * true push pending frames to setup probe timer etc. 2492 */ 2493 void tcp_push_one(struct sock *sk, unsigned int mss_now) 2494 { 2495 struct sk_buff *skb = tcp_send_head(sk); 2496 2497 BUG_ON(!skb || skb->len < mss_now); 2498 2499 tcp_write_xmit(sk, mss_now, TCP_NAGLE_PUSH, 1, sk->sk_allocation); 2500 } 2501 2502 /* This function returns the amount that we can raise the 2503 * usable window based on the following constraints 2504 * 2505 * 1. The window can never be shrunk once it is offered (RFC 793) 2506 * 2. We limit memory per socket 2507 * 2508 * RFC 1122: 2509 * "the suggested [SWS] avoidance algorithm for the receiver is to keep 2510 * RECV.NEXT + RCV.WIN fixed until: 2511 * RCV.BUFF - RCV.USER - RCV.WINDOW >= min(1/2 RCV.BUFF, MSS)" 2512 * 2513 * i.e. don't raise the right edge of the window until you can raise 2514 * it at least MSS bytes. 2515 * 2516 * Unfortunately, the recommended algorithm breaks header prediction, 2517 * since header prediction assumes th->window stays fixed. 2518 * 2519 * Strictly speaking, keeping th->window fixed violates the receiver 2520 * side SWS prevention criteria. The problem is that under this rule 2521 * a stream of single byte packets will cause the right side of the 2522 * window to always advance by a single byte. 2523 * 2524 * Of course, if the sender implements sender side SWS prevention 2525 * then this will not be a problem. 2526 * 2527 * BSD seems to make the following compromise: 2528 * 2529 * If the free space is less than the 1/4 of the maximum 2530 * space available and the free space is less than 1/2 mss, 2531 * then set the window to 0. 2532 * [ Actually, bsd uses MSS and 1/4 of maximal _window_ ] 2533 * Otherwise, just prevent the window from shrinking 2534 * and from being larger than the largest representable value. 2535 * 2536 * This prevents incremental opening of the window in the regime 2537 * where TCP is limited by the speed of the reader side taking 2538 * data out of the TCP receive queue. It does nothing about 2539 * those cases where the window is constrained on the sender side 2540 * because the pipeline is full. 2541 * 2542 * BSD also seems to "accidentally" limit itself to windows that are a 2543 * multiple of MSS, at least until the free space gets quite small. 2544 * This would appear to be a side effect of the mbuf implementation. 2545 * Combining these two algorithms results in the observed behavior 2546 * of having a fixed window size at almost all times. 2547 * 2548 * Below we obtain similar behavior by forcing the offered window to 2549 * a multiple of the mss when it is feasible to do so. 2550 * 2551 * Note, we don't "adjust" for TIMESTAMP or SACK option bytes. 2552 * Regular options like TIMESTAMP are taken into account. 2553 */ 2554 u32 __tcp_select_window(struct sock *sk) 2555 { 2556 struct inet_connection_sock *icsk = inet_csk(sk); 2557 struct tcp_sock *tp = tcp_sk(sk); 2558 /* MSS for the peer's data. Previous versions used mss_clamp 2559 * here. I don't know if the value based on our guesses 2560 * of peer's MSS is better for the performance. It's more correct 2561 * but may be worse for the performance because of rcv_mss 2562 * fluctuations. --SAW 1998/11/1 2563 */ 2564 int mss = icsk->icsk_ack.rcv_mss; 2565 int free_space = tcp_space(sk); 2566 int allowed_space = tcp_full_space(sk); 2567 int full_space = min_t(int, tp->window_clamp, allowed_space); 2568 int window; 2569 2570 if (unlikely(mss > full_space)) { 2571 mss = full_space; 2572 if (mss <= 0) 2573 return 0; 2574 } 2575 if (free_space < (full_space >> 1)) { 2576 icsk->icsk_ack.quick = 0; 2577 2578 if (tcp_under_memory_pressure(sk)) 2579 tp->rcv_ssthresh = min(tp->rcv_ssthresh, 2580 4U * tp->advmss); 2581 2582 /* free_space might become our new window, make sure we don't 2583 * increase it due to wscale. 2584 */ 2585 free_space = round_down(free_space, 1 << tp->rx_opt.rcv_wscale); 2586 2587 /* if free space is less than mss estimate, or is below 1/16th 2588 * of the maximum allowed, try to move to zero-window, else 2589 * tcp_clamp_window() will grow rcv buf up to tcp_rmem[2], and 2590 * new incoming data is dropped due to memory limits. 2591 * With large window, mss test triggers way too late in order 2592 * to announce zero window in time before rmem limit kicks in. 2593 */ 2594 if (free_space < (allowed_space >> 4) || free_space < mss) 2595 return 0; 2596 } 2597 2598 if (free_space > tp->rcv_ssthresh) 2599 free_space = tp->rcv_ssthresh; 2600 2601 /* Don't do rounding if we are using window scaling, since the 2602 * scaled window will not line up with the MSS boundary anyway. 2603 */ 2604 if (tp->rx_opt.rcv_wscale) { 2605 window = free_space; 2606 2607 /* Advertise enough space so that it won't get scaled away. 2608 * Import case: prevent zero window announcement if 2609 * 1<<rcv_wscale > mss. 2610 */ 2611 window = ALIGN(window, (1 << tp->rx_opt.rcv_wscale)); 2612 } else { 2613 window = tp->rcv_wnd; 2614 /* Get the largest window that is a nice multiple of mss. 2615 * Window clamp already applied above. 2616 * If our current window offering is within 1 mss of the 2617 * free space we just keep it. This prevents the divide 2618 * and multiply from happening most of the time. 2619 * We also don't do any window rounding when the free space 2620 * is too small. 2621 */ 2622 if (window <= free_space - mss || window > free_space) 2623 window = rounddown(free_space, mss); 2624 else if (mss == full_space && 2625 free_space > window + (full_space >> 1)) 2626 window = free_space; 2627 } 2628 2629 return window; 2630 } 2631 2632 void tcp_skb_collapse_tstamp(struct sk_buff *skb, 2633 const struct sk_buff *next_skb) 2634 { 2635 if (unlikely(tcp_has_tx_tstamp(next_skb))) { 2636 const struct skb_shared_info *next_shinfo = 2637 skb_shinfo(next_skb); 2638 struct skb_shared_info *shinfo = skb_shinfo(skb); 2639 2640 shinfo->tx_flags |= next_shinfo->tx_flags & SKBTX_ANY_TSTAMP; 2641 shinfo->tskey = next_shinfo->tskey; 2642 TCP_SKB_CB(skb)->txstamp_ack |= 2643 TCP_SKB_CB(next_skb)->txstamp_ack; 2644 } 2645 } 2646 2647 /* Collapses two adjacent SKB's during retransmission. */ 2648 static bool tcp_collapse_retrans(struct sock *sk, struct sk_buff *skb) 2649 { 2650 struct tcp_sock *tp = tcp_sk(sk); 2651 struct sk_buff *next_skb = tcp_write_queue_next(sk, skb); 2652 int skb_size, next_skb_size; 2653 2654 skb_size = skb->len; 2655 next_skb_size = next_skb->len; 2656 2657 BUG_ON(tcp_skb_pcount(skb) != 1 || tcp_skb_pcount(next_skb) != 1); 2658 2659 if (next_skb_size) { 2660 if (next_skb_size <= skb_availroom(skb)) 2661 skb_copy_bits(next_skb, 0, skb_put(skb, next_skb_size), 2662 next_skb_size); 2663 else if (!skb_shift(skb, next_skb, next_skb_size)) 2664 return false; 2665 } 2666 tcp_highest_sack_combine(sk, next_skb, skb); 2667 2668 tcp_unlink_write_queue(next_skb, sk); 2669 2670 if (next_skb->ip_summed == CHECKSUM_PARTIAL) 2671 skb->ip_summed = CHECKSUM_PARTIAL; 2672 2673 if (skb->ip_summed != CHECKSUM_PARTIAL) 2674 skb->csum = csum_block_add(skb->csum, next_skb->csum, skb_size); 2675 2676 /* Update sequence range on original skb. */ 2677 TCP_SKB_CB(skb)->end_seq = TCP_SKB_CB(next_skb)->end_seq; 2678 2679 /* Merge over control information. This moves PSH/FIN etc. over */ 2680 TCP_SKB_CB(skb)->tcp_flags |= TCP_SKB_CB(next_skb)->tcp_flags; 2681 2682 /* All done, get rid of second SKB and account for it so 2683 * packet counting does not break. 2684 */ 2685 TCP_SKB_CB(skb)->sacked |= TCP_SKB_CB(next_skb)->sacked & TCPCB_EVER_RETRANS; 2686 TCP_SKB_CB(skb)->eor = TCP_SKB_CB(next_skb)->eor; 2687 2688 /* changed transmit queue under us so clear hints */ 2689 tcp_clear_retrans_hints_partial(tp); 2690 if (next_skb == tp->retransmit_skb_hint) 2691 tp->retransmit_skb_hint = skb; 2692 2693 tcp_adjust_pcount(sk, next_skb, tcp_skb_pcount(next_skb)); 2694 2695 tcp_skb_collapse_tstamp(skb, next_skb); 2696 2697 sk_wmem_free_skb(sk, next_skb); 2698 return true; 2699 } 2700 2701 /* Check if coalescing SKBs is legal. */ 2702 static bool tcp_can_collapse(const struct sock *sk, const struct sk_buff *skb) 2703 { 2704 if (tcp_skb_pcount(skb) > 1) 2705 return false; 2706 if (skb_cloned(skb)) 2707 return false; 2708 if (skb == tcp_send_head(sk)) 2709 return false; 2710 /* Some heuristics for collapsing over SACK'd could be invented */ 2711 if (TCP_SKB_CB(skb)->sacked & TCPCB_SACKED_ACKED) 2712 return false; 2713 2714 return true; 2715 } 2716 2717 /* Collapse packets in the retransmit queue to make to create 2718 * less packets on the wire. This is only done on retransmission. 2719 */ 2720 static void tcp_retrans_try_collapse(struct sock *sk, struct sk_buff *to, 2721 int space) 2722 { 2723 struct tcp_sock *tp = tcp_sk(sk); 2724 struct sk_buff *skb = to, *tmp; 2725 bool first = true; 2726 2727 if (!sysctl_tcp_retrans_collapse) 2728 return; 2729 if (TCP_SKB_CB(skb)->tcp_flags & TCPHDR_SYN) 2730 return; 2731 2732 tcp_for_write_queue_from_safe(skb, tmp, sk) { 2733 if (!tcp_can_collapse(sk, skb)) 2734 break; 2735 2736 if (!tcp_skb_can_collapse_to(to)) 2737 break; 2738 2739 space -= skb->len; 2740 2741 if (first) { 2742 first = false; 2743 continue; 2744 } 2745 2746 if (space < 0) 2747 break; 2748 2749 if (after(TCP_SKB_CB(skb)->end_seq, tcp_wnd_end(tp))) 2750 break; 2751 2752 if (!tcp_collapse_retrans(sk, to)) 2753 break; 2754 } 2755 } 2756 2757 /* This retransmits one SKB. Policy decisions and retransmit queue 2758 * state updates are done by the caller. Returns non-zero if an 2759 * error occurred which prevented the send. 2760 */ 2761 int __tcp_retransmit_skb(struct sock *sk, struct sk_buff *skb, int segs) 2762 { 2763 struct inet_connection_sock *icsk = inet_csk(sk); 2764 struct tcp_sock *tp = tcp_sk(sk); 2765 unsigned int cur_mss; 2766 int diff, len, err; 2767 2768 2769 /* Inconclusive MTU probe */ 2770 if (icsk->icsk_mtup.probe_size) 2771 icsk->icsk_mtup.probe_size = 0; 2772 2773 /* Do not sent more than we queued. 1/4 is reserved for possible 2774 * copying overhead: fragmentation, tunneling, mangling etc. 2775 */ 2776 if (refcount_read(&sk->sk_wmem_alloc) > 2777 min_t(u32, sk->sk_wmem_queued + (sk->sk_wmem_queued >> 2), 2778 sk->sk_sndbuf)) 2779 return -EAGAIN; 2780 2781 if (skb_still_in_host_queue(sk, skb)) 2782 return -EBUSY; 2783 2784 if (before(TCP_SKB_CB(skb)->seq, tp->snd_una)) { 2785 if (before(TCP_SKB_CB(skb)->end_seq, tp->snd_una)) 2786 BUG(); 2787 if (tcp_trim_head(sk, skb, tp->snd_una - TCP_SKB_CB(skb)->seq)) 2788 return -ENOMEM; 2789 } 2790 2791 if (inet_csk(sk)->icsk_af_ops->rebuild_header(sk)) 2792 return -EHOSTUNREACH; /* Routing failure or similar. */ 2793 2794 cur_mss = tcp_current_mss(sk); 2795 2796 /* If receiver has shrunk his window, and skb is out of 2797 * new window, do not retransmit it. The exception is the 2798 * case, when window is shrunk to zero. In this case 2799 * our retransmit serves as a zero window probe. 2800 */ 2801 if (!before(TCP_SKB_CB(skb)->seq, tcp_wnd_end(tp)) && 2802 TCP_SKB_CB(skb)->seq != tp->snd_una) 2803 return -EAGAIN; 2804 2805 len = cur_mss * segs; 2806 if (skb->len > len) { 2807 if (tcp_fragment(sk, skb, len, cur_mss, GFP_ATOMIC)) 2808 return -ENOMEM; /* We'll try again later. */ 2809 } else { 2810 if (skb_unclone(skb, GFP_ATOMIC)) 2811 return -ENOMEM; 2812 2813 diff = tcp_skb_pcount(skb); 2814 tcp_set_skb_tso_segs(skb, cur_mss); 2815 diff -= tcp_skb_pcount(skb); 2816 if (diff) 2817 tcp_adjust_pcount(sk, skb, diff); 2818 if (skb->len < cur_mss) 2819 tcp_retrans_try_collapse(sk, skb, cur_mss); 2820 } 2821 2822 /* RFC3168, section 6.1.1.1. ECN fallback */ 2823 if ((TCP_SKB_CB(skb)->tcp_flags & TCPHDR_SYN_ECN) == TCPHDR_SYN_ECN) 2824 tcp_ecn_clear_syn(sk, skb); 2825 2826 /* Update global and local TCP statistics. */ 2827 segs = tcp_skb_pcount(skb); 2828 TCP_ADD_STATS(sock_net(sk), TCP_MIB_RETRANSSEGS, segs); 2829 if (TCP_SKB_CB(skb)->tcp_flags & TCPHDR_SYN) 2830 __NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPSYNRETRANS); 2831 tp->total_retrans += segs; 2832 2833 /* make sure skb->data is aligned on arches that require it 2834 * and check if ack-trimming & collapsing extended the headroom 2835 * beyond what csum_start can cover. 2836 */ 2837 if (unlikely((NET_IP_ALIGN && ((unsigned long)skb->data & 3)) || 2838 skb_headroom(skb) >= 0xFFFF)) { 2839 struct sk_buff *nskb; 2840 2841 nskb = __pskb_copy(skb, MAX_TCP_HEADER, GFP_ATOMIC); 2842 err = nskb ? tcp_transmit_skb(sk, nskb, 0, GFP_ATOMIC) : 2843 -ENOBUFS; 2844 if (!err) 2845 skb->skb_mstamp = tp->tcp_mstamp; 2846 } else { 2847 err = tcp_transmit_skb(sk, skb, 1, GFP_ATOMIC); 2848 } 2849 2850 if (likely(!err)) { 2851 TCP_SKB_CB(skb)->sacked |= TCPCB_EVER_RETRANS; 2852 } else if (err != -EBUSY) { 2853 NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPRETRANSFAIL); 2854 } 2855 return err; 2856 } 2857 2858 int tcp_retransmit_skb(struct sock *sk, struct sk_buff *skb, int segs) 2859 { 2860 struct tcp_sock *tp = tcp_sk(sk); 2861 int err = __tcp_retransmit_skb(sk, skb, segs); 2862 2863 if (err == 0) { 2864 #if FASTRETRANS_DEBUG > 0 2865 if (TCP_SKB_CB(skb)->sacked & TCPCB_SACKED_RETRANS) { 2866 net_dbg_ratelimited("retrans_out leaked\n"); 2867 } 2868 #endif 2869 TCP_SKB_CB(skb)->sacked |= TCPCB_RETRANS; 2870 tp->retrans_out += tcp_skb_pcount(skb); 2871 2872 /* Save stamp of the first retransmit. */ 2873 if (!tp->retrans_stamp) 2874 tp->retrans_stamp = tcp_skb_timestamp(skb); 2875 2876 } 2877 2878 if (tp->undo_retrans < 0) 2879 tp->undo_retrans = 0; 2880 tp->undo_retrans += tcp_skb_pcount(skb); 2881 return err; 2882 } 2883 2884 /* This gets called after a retransmit timeout, and the initially 2885 * retransmitted data is acknowledged. It tries to continue 2886 * resending the rest of the retransmit queue, until either 2887 * we've sent it all or the congestion window limit is reached. 2888 * If doing SACK, the first ACK which comes back for a timeout 2889 * based retransmit packet might feed us FACK information again. 2890 * If so, we use it to avoid unnecessarily retransmissions. 2891 */ 2892 void tcp_xmit_retransmit_queue(struct sock *sk) 2893 { 2894 const struct inet_connection_sock *icsk = inet_csk(sk); 2895 struct tcp_sock *tp = tcp_sk(sk); 2896 struct sk_buff *skb; 2897 struct sk_buff *hole = NULL; 2898 u32 max_segs; 2899 int mib_idx; 2900 2901 if (!tp->packets_out) 2902 return; 2903 2904 if (tp->retransmit_skb_hint) { 2905 skb = tp->retransmit_skb_hint; 2906 } else { 2907 skb = tcp_write_queue_head(sk); 2908 } 2909 2910 max_segs = tcp_tso_segs(sk, tcp_current_mss(sk)); 2911 tcp_for_write_queue_from(skb, sk) { 2912 __u8 sacked; 2913 int segs; 2914 2915 if (skb == tcp_send_head(sk)) 2916 break; 2917 2918 if (tcp_pacing_check(sk)) 2919 break; 2920 2921 /* we could do better than to assign each time */ 2922 if (!hole) 2923 tp->retransmit_skb_hint = skb; 2924 2925 segs = tp->snd_cwnd - tcp_packets_in_flight(tp); 2926 if (segs <= 0) 2927 return; 2928 sacked = TCP_SKB_CB(skb)->sacked; 2929 /* In case tcp_shift_skb_data() have aggregated large skbs, 2930 * we need to make sure not sending too bigs TSO packets 2931 */ 2932 segs = min_t(int, segs, max_segs); 2933 2934 if (tp->retrans_out >= tp->lost_out) { 2935 break; 2936 } else if (!(sacked & TCPCB_LOST)) { 2937 if (!hole && !(sacked & (TCPCB_SACKED_RETRANS|TCPCB_SACKED_ACKED))) 2938 hole = skb; 2939 continue; 2940 2941 } else { 2942 if (icsk->icsk_ca_state != TCP_CA_Loss) 2943 mib_idx = LINUX_MIB_TCPFASTRETRANS; 2944 else 2945 mib_idx = LINUX_MIB_TCPSLOWSTARTRETRANS; 2946 } 2947 2948 if (sacked & (TCPCB_SACKED_ACKED|TCPCB_SACKED_RETRANS)) 2949 continue; 2950 2951 if (tcp_small_queue_check(sk, skb, 1)) 2952 return; 2953 2954 if (tcp_retransmit_skb(sk, skb, segs)) 2955 return; 2956 2957 NET_ADD_STATS(sock_net(sk), mib_idx, tcp_skb_pcount(skb)); 2958 2959 if (tcp_in_cwnd_reduction(sk)) 2960 tp->prr_out += tcp_skb_pcount(skb); 2961 2962 if (skb == tcp_write_queue_head(sk) && 2963 icsk->icsk_pending != ICSK_TIME_REO_TIMEOUT) 2964 inet_csk_reset_xmit_timer(sk, ICSK_TIME_RETRANS, 2965 inet_csk(sk)->icsk_rto, 2966 TCP_RTO_MAX); 2967 } 2968 } 2969 2970 /* We allow to exceed memory limits for FIN packets to expedite 2971 * connection tear down and (memory) recovery. 2972 * Otherwise tcp_send_fin() could be tempted to either delay FIN 2973 * or even be forced to close flow without any FIN. 2974 * In general, we want to allow one skb per socket to avoid hangs 2975 * with edge trigger epoll() 2976 */ 2977 void sk_forced_mem_schedule(struct sock *sk, int size) 2978 { 2979 int amt; 2980 2981 if (size <= sk->sk_forward_alloc) 2982 return; 2983 amt = sk_mem_pages(size); 2984 sk->sk_forward_alloc += amt * SK_MEM_QUANTUM; 2985 sk_memory_allocated_add(sk, amt); 2986 2987 if (mem_cgroup_sockets_enabled && sk->sk_memcg) 2988 mem_cgroup_charge_skmem(sk->sk_memcg, amt); 2989 } 2990 2991 /* Send a FIN. The caller locks the socket for us. 2992 * We should try to send a FIN packet really hard, but eventually give up. 2993 */ 2994 void tcp_send_fin(struct sock *sk) 2995 { 2996 struct sk_buff *skb, *tskb = tcp_write_queue_tail(sk); 2997 struct tcp_sock *tp = tcp_sk(sk); 2998 2999 /* Optimization, tack on the FIN if we have one skb in write queue and 3000 * this skb was not yet sent, or we are under memory pressure. 3001 * Note: in the latter case, FIN packet will be sent after a timeout, 3002 * as TCP stack thinks it has already been transmitted. 3003 */ 3004 if (tskb && (tcp_send_head(sk) || tcp_under_memory_pressure(sk))) { 3005 coalesce: 3006 TCP_SKB_CB(tskb)->tcp_flags |= TCPHDR_FIN; 3007 TCP_SKB_CB(tskb)->end_seq++; 3008 tp->write_seq++; 3009 if (!tcp_send_head(sk)) { 3010 /* This means tskb was already sent. 3011 * Pretend we included the FIN on previous transmit. 3012 * We need to set tp->snd_nxt to the value it would have 3013 * if FIN had been sent. This is because retransmit path 3014 * does not change tp->snd_nxt. 3015 */ 3016 tp->snd_nxt++; 3017 return; 3018 } 3019 } else { 3020 skb = alloc_skb_fclone(MAX_TCP_HEADER, sk->sk_allocation); 3021 if (unlikely(!skb)) { 3022 if (tskb) 3023 goto coalesce; 3024 return; 3025 } 3026 skb_reserve(skb, MAX_TCP_HEADER); 3027 sk_forced_mem_schedule(sk, skb->truesize); 3028 /* FIN eats a sequence byte, write_seq advanced by tcp_queue_skb(). */ 3029 tcp_init_nondata_skb(skb, tp->write_seq, 3030 TCPHDR_ACK | TCPHDR_FIN); 3031 tcp_queue_skb(sk, skb); 3032 } 3033 __tcp_push_pending_frames(sk, tcp_current_mss(sk), TCP_NAGLE_OFF); 3034 } 3035 3036 /* We get here when a process closes a file descriptor (either due to 3037 * an explicit close() or as a byproduct of exit()'ing) and there 3038 * was unread data in the receive queue. This behavior is recommended 3039 * by RFC 2525, section 2.17. -DaveM 3040 */ 3041 void tcp_send_active_reset(struct sock *sk, gfp_t priority) 3042 { 3043 struct sk_buff *skb; 3044 3045 TCP_INC_STATS(sock_net(sk), TCP_MIB_OUTRSTS); 3046 3047 /* NOTE: No TCP options attached and we never retransmit this. */ 3048 skb = alloc_skb(MAX_TCP_HEADER, priority); 3049 if (!skb) { 3050 NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPABORTFAILED); 3051 return; 3052 } 3053 3054 /* Reserve space for headers and prepare control bits. */ 3055 skb_reserve(skb, MAX_TCP_HEADER); 3056 tcp_init_nondata_skb(skb, tcp_acceptable_seq(sk), 3057 TCPHDR_ACK | TCPHDR_RST); 3058 tcp_mstamp_refresh(tcp_sk(sk)); 3059 /* Send it off. */ 3060 if (tcp_transmit_skb(sk, skb, 0, priority)) 3061 NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPABORTFAILED); 3062 } 3063 3064 /* Send a crossed SYN-ACK during socket establishment. 3065 * WARNING: This routine must only be called when we have already sent 3066 * a SYN packet that crossed the incoming SYN that caused this routine 3067 * to get called. If this assumption fails then the initial rcv_wnd 3068 * and rcv_wscale values will not be correct. 3069 */ 3070 int tcp_send_synack(struct sock *sk) 3071 { 3072 struct sk_buff *skb; 3073 3074 skb = tcp_write_queue_head(sk); 3075 if (!skb || !(TCP_SKB_CB(skb)->tcp_flags & TCPHDR_SYN)) { 3076 pr_debug("%s: wrong queue state\n", __func__); 3077 return -EFAULT; 3078 } 3079 if (!(TCP_SKB_CB(skb)->tcp_flags & TCPHDR_ACK)) { 3080 if (skb_cloned(skb)) { 3081 struct sk_buff *nskb = skb_copy(skb, GFP_ATOMIC); 3082 if (!nskb) 3083 return -ENOMEM; 3084 tcp_unlink_write_queue(skb, sk); 3085 __skb_header_release(nskb); 3086 __tcp_add_write_queue_head(sk, nskb); 3087 sk_wmem_free_skb(sk, skb); 3088 sk->sk_wmem_queued += nskb->truesize; 3089 sk_mem_charge(sk, nskb->truesize); 3090 skb = nskb; 3091 } 3092 3093 TCP_SKB_CB(skb)->tcp_flags |= TCPHDR_ACK; 3094 tcp_ecn_send_synack(sk, skb); 3095 } 3096 return tcp_transmit_skb(sk, skb, 1, GFP_ATOMIC); 3097 } 3098 3099 /** 3100 * tcp_make_synack - Prepare a SYN-ACK. 3101 * sk: listener socket 3102 * dst: dst entry attached to the SYNACK 3103 * req: request_sock pointer 3104 * 3105 * Allocate one skb and build a SYNACK packet. 3106 * @dst is consumed : Caller should not use it again. 3107 */ 3108 struct sk_buff *tcp_make_synack(const struct sock *sk, struct dst_entry *dst, 3109 struct request_sock *req, 3110 struct tcp_fastopen_cookie *foc, 3111 enum tcp_synack_type synack_type) 3112 { 3113 struct inet_request_sock *ireq = inet_rsk(req); 3114 const struct tcp_sock *tp = tcp_sk(sk); 3115 struct tcp_md5sig_key *md5 = NULL; 3116 struct tcp_out_options opts; 3117 struct sk_buff *skb; 3118 int tcp_header_size; 3119 struct tcphdr *th; 3120 int mss; 3121 3122 skb = alloc_skb(MAX_TCP_HEADER, GFP_ATOMIC); 3123 if (unlikely(!skb)) { 3124 dst_release(dst); 3125 return NULL; 3126 } 3127 /* Reserve space for headers. */ 3128 skb_reserve(skb, MAX_TCP_HEADER); 3129 3130 switch (synack_type) { 3131 case TCP_SYNACK_NORMAL: 3132 skb_set_owner_w(skb, req_to_sk(req)); 3133 break; 3134 case TCP_SYNACK_COOKIE: 3135 /* Under synflood, we do not attach skb to a socket, 3136 * to avoid false sharing. 3137 */ 3138 break; 3139 case TCP_SYNACK_FASTOPEN: 3140 /* sk is a const pointer, because we want to express multiple 3141 * cpu might call us concurrently. 3142 * sk->sk_wmem_alloc in an atomic, we can promote to rw. 3143 */ 3144 skb_set_owner_w(skb, (struct sock *)sk); 3145 break; 3146 } 3147 skb_dst_set(skb, dst); 3148 3149 mss = tcp_mss_clamp(tp, dst_metric_advmss(dst)); 3150 3151 memset(&opts, 0, sizeof(opts)); 3152 #ifdef CONFIG_SYN_COOKIES 3153 if (unlikely(req->cookie_ts)) 3154 skb->skb_mstamp = cookie_init_timestamp(req); 3155 else 3156 #endif 3157 skb->skb_mstamp = tcp_clock_us(); 3158 3159 #ifdef CONFIG_TCP_MD5SIG 3160 rcu_read_lock(); 3161 md5 = tcp_rsk(req)->af_specific->req_md5_lookup(sk, req_to_sk(req)); 3162 #endif 3163 skb_set_hash(skb, tcp_rsk(req)->txhash, PKT_HASH_TYPE_L4); 3164 tcp_header_size = tcp_synack_options(req, mss, skb, &opts, md5, foc) + 3165 sizeof(*th); 3166 3167 skb_push(skb, tcp_header_size); 3168 skb_reset_transport_header(skb); 3169 3170 th = (struct tcphdr *)skb->data; 3171 memset(th, 0, sizeof(struct tcphdr)); 3172 th->syn = 1; 3173 th->ack = 1; 3174 tcp_ecn_make_synack(req, th); 3175 th->source = htons(ireq->ir_num); 3176 th->dest = ireq->ir_rmt_port; 3177 skb->mark = ireq->ir_mark; 3178 /* Setting of flags are superfluous here for callers (and ECE is 3179 * not even correctly set) 3180 */ 3181 tcp_init_nondata_skb(skb, tcp_rsk(req)->snt_isn, 3182 TCPHDR_SYN | TCPHDR_ACK); 3183 3184 th->seq = htonl(TCP_SKB_CB(skb)->seq); 3185 /* XXX data is queued and acked as is. No buffer/window check */ 3186 th->ack_seq = htonl(tcp_rsk(req)->rcv_nxt); 3187 3188 /* RFC1323: The window in SYN & SYN/ACK segments is never scaled. */ 3189 th->window = htons(min(req->rsk_rcv_wnd, 65535U)); 3190 tcp_options_write((__be32 *)(th + 1), NULL, &opts); 3191 th->doff = (tcp_header_size >> 2); 3192 __TCP_INC_STATS(sock_net(sk), TCP_MIB_OUTSEGS); 3193 3194 #ifdef CONFIG_TCP_MD5SIG 3195 /* Okay, we have all we need - do the md5 hash if needed */ 3196 if (md5) 3197 tcp_rsk(req)->af_specific->calc_md5_hash(opts.hash_location, 3198 md5, req_to_sk(req), skb); 3199 rcu_read_unlock(); 3200 #endif 3201 3202 /* Do not fool tcpdump (if any), clean our debris */ 3203 skb->tstamp = 0; 3204 return skb; 3205 } 3206 EXPORT_SYMBOL(tcp_make_synack); 3207 3208 static void tcp_ca_dst_init(struct sock *sk, const struct dst_entry *dst) 3209 { 3210 struct inet_connection_sock *icsk = inet_csk(sk); 3211 const struct tcp_congestion_ops *ca; 3212 u32 ca_key = dst_metric(dst, RTAX_CC_ALGO); 3213 3214 if (ca_key == TCP_CA_UNSPEC) 3215 return; 3216 3217 rcu_read_lock(); 3218 ca = tcp_ca_find_key(ca_key); 3219 if (likely(ca && try_module_get(ca->owner))) { 3220 module_put(icsk->icsk_ca_ops->owner); 3221 icsk->icsk_ca_dst_locked = tcp_ca_dst_locked(dst); 3222 icsk->icsk_ca_ops = ca; 3223 } 3224 rcu_read_unlock(); 3225 } 3226 3227 /* Do all connect socket setups that can be done AF independent. */ 3228 static void tcp_connect_init(struct sock *sk) 3229 { 3230 const struct dst_entry *dst = __sk_dst_get(sk); 3231 struct tcp_sock *tp = tcp_sk(sk); 3232 __u8 rcv_wscale; 3233 u32 rcv_wnd; 3234 3235 /* We'll fix this up when we get a response from the other end. 3236 * See tcp_input.c:tcp_rcv_state_process case TCP_SYN_SENT. 3237 */ 3238 tp->tcp_header_len = sizeof(struct tcphdr); 3239 if (sock_net(sk)->ipv4.sysctl_tcp_timestamps) 3240 tp->tcp_header_len += TCPOLEN_TSTAMP_ALIGNED; 3241 3242 #ifdef CONFIG_TCP_MD5SIG 3243 if (tp->af_specific->md5_lookup(sk, sk)) 3244 tp->tcp_header_len += TCPOLEN_MD5SIG_ALIGNED; 3245 #endif 3246 3247 /* If user gave his TCP_MAXSEG, record it to clamp */ 3248 if (tp->rx_opt.user_mss) 3249 tp->rx_opt.mss_clamp = tp->rx_opt.user_mss; 3250 tp->max_window = 0; 3251 tcp_mtup_init(sk); 3252 tcp_sync_mss(sk, dst_mtu(dst)); 3253 3254 tcp_ca_dst_init(sk, dst); 3255 3256 if (!tp->window_clamp) 3257 tp->window_clamp = dst_metric(dst, RTAX_WINDOW); 3258 tp->advmss = tcp_mss_clamp(tp, dst_metric_advmss(dst)); 3259 3260 tcp_initialize_rcv_mss(sk); 3261 3262 /* limit the window selection if the user enforce a smaller rx buffer */ 3263 if (sk->sk_userlocks & SOCK_RCVBUF_LOCK && 3264 (tp->window_clamp > tcp_full_space(sk) || tp->window_clamp == 0)) 3265 tp->window_clamp = tcp_full_space(sk); 3266 3267 rcv_wnd = tcp_rwnd_init_bpf(sk); 3268 if (rcv_wnd == 0) 3269 rcv_wnd = dst_metric(dst, RTAX_INITRWND); 3270 3271 tcp_select_initial_window(tcp_full_space(sk), 3272 tp->advmss - (tp->rx_opt.ts_recent_stamp ? tp->tcp_header_len - sizeof(struct tcphdr) : 0), 3273 &tp->rcv_wnd, 3274 &tp->window_clamp, 3275 sock_net(sk)->ipv4.sysctl_tcp_window_scaling, 3276 &rcv_wscale, 3277 rcv_wnd); 3278 3279 tp->rx_opt.rcv_wscale = rcv_wscale; 3280 tp->rcv_ssthresh = tp->rcv_wnd; 3281 3282 sk->sk_err = 0; 3283 sock_reset_flag(sk, SOCK_DONE); 3284 tp->snd_wnd = 0; 3285 tcp_init_wl(tp, 0); 3286 tp->snd_una = tp->write_seq; 3287 tp->snd_sml = tp->write_seq; 3288 tp->snd_up = tp->write_seq; 3289 tp->snd_nxt = tp->write_seq; 3290 3291 if (likely(!tp->repair)) 3292 tp->rcv_nxt = 0; 3293 else 3294 tp->rcv_tstamp = tcp_jiffies32; 3295 tp->rcv_wup = tp->rcv_nxt; 3296 tp->copied_seq = tp->rcv_nxt; 3297 3298 inet_csk(sk)->icsk_rto = tcp_timeout_init(sk); 3299 inet_csk(sk)->icsk_retransmits = 0; 3300 tcp_clear_retrans(tp); 3301 } 3302 3303 static void tcp_connect_queue_skb(struct sock *sk, struct sk_buff *skb) 3304 { 3305 struct tcp_sock *tp = tcp_sk(sk); 3306 struct tcp_skb_cb *tcb = TCP_SKB_CB(skb); 3307 3308 tcb->end_seq += skb->len; 3309 __skb_header_release(skb); 3310 __tcp_add_write_queue_tail(sk, skb); 3311 sk->sk_wmem_queued += skb->truesize; 3312 sk_mem_charge(sk, skb->truesize); 3313 tp->write_seq = tcb->end_seq; 3314 tp->packets_out += tcp_skb_pcount(skb); 3315 } 3316 3317 /* Build and send a SYN with data and (cached) Fast Open cookie. However, 3318 * queue a data-only packet after the regular SYN, such that regular SYNs 3319 * are retransmitted on timeouts. Also if the remote SYN-ACK acknowledges 3320 * only the SYN sequence, the data are retransmitted in the first ACK. 3321 * If cookie is not cached or other error occurs, falls back to send a 3322 * regular SYN with Fast Open cookie request option. 3323 */ 3324 static int tcp_send_syn_data(struct sock *sk, struct sk_buff *syn) 3325 { 3326 struct tcp_sock *tp = tcp_sk(sk); 3327 struct tcp_fastopen_request *fo = tp->fastopen_req; 3328 int space, err = 0; 3329 struct sk_buff *syn_data; 3330 3331 tp->rx_opt.mss_clamp = tp->advmss; /* If MSS is not cached */ 3332 if (!tcp_fastopen_cookie_check(sk, &tp->rx_opt.mss_clamp, &fo->cookie)) 3333 goto fallback; 3334 3335 /* MSS for SYN-data is based on cached MSS and bounded by PMTU and 3336 * user-MSS. Reserve maximum option space for middleboxes that add 3337 * private TCP options. The cost is reduced data space in SYN :( 3338 */ 3339 tp->rx_opt.mss_clamp = tcp_mss_clamp(tp, tp->rx_opt.mss_clamp); 3340 3341 space = __tcp_mtu_to_mss(sk, inet_csk(sk)->icsk_pmtu_cookie) - 3342 MAX_TCP_OPTION_SPACE; 3343 3344 space = min_t(size_t, space, fo->size); 3345 3346 /* limit to order-0 allocations */ 3347 space = min_t(size_t, space, SKB_MAX_HEAD(MAX_TCP_HEADER)); 3348 3349 syn_data = sk_stream_alloc_skb(sk, space, sk->sk_allocation, false); 3350 if (!syn_data) 3351 goto fallback; 3352 syn_data->ip_summed = CHECKSUM_PARTIAL; 3353 memcpy(syn_data->cb, syn->cb, sizeof(syn->cb)); 3354 if (space) { 3355 int copied = copy_from_iter(skb_put(syn_data, space), space, 3356 &fo->data->msg_iter); 3357 if (unlikely(!copied)) { 3358 kfree_skb(syn_data); 3359 goto fallback; 3360 } 3361 if (copied != space) { 3362 skb_trim(syn_data, copied); 3363 space = copied; 3364 } 3365 } 3366 /* No more data pending in inet_wait_for_connect() */ 3367 if (space == fo->size) 3368 fo->data = NULL; 3369 fo->copied = space; 3370 3371 tcp_connect_queue_skb(sk, syn_data); 3372 if (syn_data->len) 3373 tcp_chrono_start(sk, TCP_CHRONO_BUSY); 3374 3375 err = tcp_transmit_skb(sk, syn_data, 1, sk->sk_allocation); 3376 3377 syn->skb_mstamp = syn_data->skb_mstamp; 3378 3379 /* Now full SYN+DATA was cloned and sent (or not), 3380 * remove the SYN from the original skb (syn_data) 3381 * we keep in write queue in case of a retransmit, as we 3382 * also have the SYN packet (with no data) in the same queue. 3383 */ 3384 TCP_SKB_CB(syn_data)->seq++; 3385 TCP_SKB_CB(syn_data)->tcp_flags = TCPHDR_ACK | TCPHDR_PSH; 3386 if (!err) { 3387 tp->syn_data = (fo->copied > 0); 3388 NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPORIGDATASENT); 3389 goto done; 3390 } 3391 3392 /* data was not sent, this is our new send_head */ 3393 sk->sk_send_head = syn_data; 3394 tp->packets_out -= tcp_skb_pcount(syn_data); 3395 3396 fallback: 3397 /* Send a regular SYN with Fast Open cookie request option */ 3398 if (fo->cookie.len > 0) 3399 fo->cookie.len = 0; 3400 err = tcp_transmit_skb(sk, syn, 1, sk->sk_allocation); 3401 if (err) 3402 tp->syn_fastopen = 0; 3403 done: 3404 fo->cookie.len = -1; /* Exclude Fast Open option for SYN retries */ 3405 return err; 3406 } 3407 3408 /* Build a SYN and send it off. */ 3409 int tcp_connect(struct sock *sk) 3410 { 3411 struct tcp_sock *tp = tcp_sk(sk); 3412 struct sk_buff *buff; 3413 int err; 3414 3415 tcp_call_bpf(sk, BPF_SOCK_OPS_TCP_CONNECT_CB); 3416 3417 if (inet_csk(sk)->icsk_af_ops->rebuild_header(sk)) 3418 return -EHOSTUNREACH; /* Routing failure or similar. */ 3419 3420 tcp_connect_init(sk); 3421 3422 if (unlikely(tp->repair)) { 3423 tcp_finish_connect(sk, NULL); 3424 return 0; 3425 } 3426 3427 buff = sk_stream_alloc_skb(sk, 0, sk->sk_allocation, true); 3428 if (unlikely(!buff)) 3429 return -ENOBUFS; 3430 3431 tcp_init_nondata_skb(buff, tp->write_seq++, TCPHDR_SYN); 3432 tcp_mstamp_refresh(tp); 3433 tp->retrans_stamp = tcp_time_stamp(tp); 3434 tcp_connect_queue_skb(sk, buff); 3435 tcp_ecn_send_syn(sk, buff); 3436 3437 /* Send off SYN; include data in Fast Open. */ 3438 err = tp->fastopen_req ? tcp_send_syn_data(sk, buff) : 3439 tcp_transmit_skb(sk, buff, 1, sk->sk_allocation); 3440 if (err == -ECONNREFUSED) 3441 return err; 3442 3443 /* We change tp->snd_nxt after the tcp_transmit_skb() call 3444 * in order to make this packet get counted in tcpOutSegs. 3445 */ 3446 tp->snd_nxt = tp->write_seq; 3447 tp->pushed_seq = tp->write_seq; 3448 buff = tcp_send_head(sk); 3449 if (unlikely(buff)) { 3450 tp->snd_nxt = TCP_SKB_CB(buff)->seq; 3451 tp->pushed_seq = TCP_SKB_CB(buff)->seq; 3452 } 3453 TCP_INC_STATS(sock_net(sk), TCP_MIB_ACTIVEOPENS); 3454 3455 /* Timer for repeating the SYN until an answer. */ 3456 inet_csk_reset_xmit_timer(sk, ICSK_TIME_RETRANS, 3457 inet_csk(sk)->icsk_rto, TCP_RTO_MAX); 3458 return 0; 3459 } 3460 EXPORT_SYMBOL(tcp_connect); 3461 3462 /* Send out a delayed ack, the caller does the policy checking 3463 * to see if we should even be here. See tcp_input.c:tcp_ack_snd_check() 3464 * for details. 3465 */ 3466 void tcp_send_delayed_ack(struct sock *sk) 3467 { 3468 struct inet_connection_sock *icsk = inet_csk(sk); 3469 int ato = icsk->icsk_ack.ato; 3470 unsigned long timeout; 3471 3472 tcp_ca_event(sk, CA_EVENT_DELAYED_ACK); 3473 3474 if (ato > TCP_DELACK_MIN) { 3475 const struct tcp_sock *tp = tcp_sk(sk); 3476 int max_ato = HZ / 2; 3477 3478 if (icsk->icsk_ack.pingpong || 3479 (icsk->icsk_ack.pending & ICSK_ACK_PUSHED)) 3480 max_ato = TCP_DELACK_MAX; 3481 3482 /* Slow path, intersegment interval is "high". */ 3483 3484 /* If some rtt estimate is known, use it to bound delayed ack. 3485 * Do not use inet_csk(sk)->icsk_rto here, use results of rtt measurements 3486 * directly. 3487 */ 3488 if (tp->srtt_us) { 3489 int rtt = max_t(int, usecs_to_jiffies(tp->srtt_us >> 3), 3490 TCP_DELACK_MIN); 3491 3492 if (rtt < max_ato) 3493 max_ato = rtt; 3494 } 3495 3496 ato = min(ato, max_ato); 3497 } 3498 3499 /* Stay within the limit we were given */ 3500 timeout = jiffies + ato; 3501 3502 /* Use new timeout only if there wasn't a older one earlier. */ 3503 if (icsk->icsk_ack.pending & ICSK_ACK_TIMER) { 3504 /* If delack timer was blocked or is about to expire, 3505 * send ACK now. 3506 */ 3507 if (icsk->icsk_ack.blocked || 3508 time_before_eq(icsk->icsk_ack.timeout, jiffies + (ato >> 2))) { 3509 tcp_send_ack(sk); 3510 return; 3511 } 3512 3513 if (!time_before(timeout, icsk->icsk_ack.timeout)) 3514 timeout = icsk->icsk_ack.timeout; 3515 } 3516 icsk->icsk_ack.pending |= ICSK_ACK_SCHED | ICSK_ACK_TIMER; 3517 icsk->icsk_ack.timeout = timeout; 3518 sk_reset_timer(sk, &icsk->icsk_delack_timer, timeout); 3519 } 3520 3521 /* This routine sends an ack and also updates the window. */ 3522 void tcp_send_ack(struct sock *sk) 3523 { 3524 struct sk_buff *buff; 3525 3526 /* If we have been reset, we may not send again. */ 3527 if (sk->sk_state == TCP_CLOSE) 3528 return; 3529 3530 tcp_ca_event(sk, CA_EVENT_NON_DELAYED_ACK); 3531 3532 /* We are not putting this on the write queue, so 3533 * tcp_transmit_skb() will set the ownership to this 3534 * sock. 3535 */ 3536 buff = alloc_skb(MAX_TCP_HEADER, 3537 sk_gfp_mask(sk, GFP_ATOMIC | __GFP_NOWARN)); 3538 if (unlikely(!buff)) { 3539 inet_csk_schedule_ack(sk); 3540 inet_csk(sk)->icsk_ack.ato = TCP_ATO_MIN; 3541 inet_csk_reset_xmit_timer(sk, ICSK_TIME_DACK, 3542 TCP_DELACK_MAX, TCP_RTO_MAX); 3543 return; 3544 } 3545 3546 /* Reserve space for headers and prepare control bits. */ 3547 skb_reserve(buff, MAX_TCP_HEADER); 3548 tcp_init_nondata_skb(buff, tcp_acceptable_seq(sk), TCPHDR_ACK); 3549 3550 /* We do not want pure acks influencing TCP Small Queues or fq/pacing 3551 * too much. 3552 * SKB_TRUESIZE(max(1 .. 66, MAX_TCP_HEADER)) is unfortunately ~784 3553 */ 3554 skb_set_tcp_pure_ack(buff); 3555 3556 /* Send it off, this clears delayed acks for us. */ 3557 tcp_transmit_skb(sk, buff, 0, (__force gfp_t)0); 3558 } 3559 EXPORT_SYMBOL_GPL(tcp_send_ack); 3560 3561 /* This routine sends a packet with an out of date sequence 3562 * number. It assumes the other end will try to ack it. 3563 * 3564 * Question: what should we make while urgent mode? 3565 * 4.4BSD forces sending single byte of data. We cannot send 3566 * out of window data, because we have SND.NXT==SND.MAX... 3567 * 3568 * Current solution: to send TWO zero-length segments in urgent mode: 3569 * one is with SEG.SEQ=SND.UNA to deliver urgent pointer, another is 3570 * out-of-date with SND.UNA-1 to probe window. 3571 */ 3572 static int tcp_xmit_probe_skb(struct sock *sk, int urgent, int mib) 3573 { 3574 struct tcp_sock *tp = tcp_sk(sk); 3575 struct sk_buff *skb; 3576 3577 /* We don't queue it, tcp_transmit_skb() sets ownership. */ 3578 skb = alloc_skb(MAX_TCP_HEADER, 3579 sk_gfp_mask(sk, GFP_ATOMIC | __GFP_NOWARN)); 3580 if (!skb) 3581 return -1; 3582 3583 /* Reserve space for headers and set control bits. */ 3584 skb_reserve(skb, MAX_TCP_HEADER); 3585 /* Use a previous sequence. This should cause the other 3586 * end to send an ack. Don't queue or clone SKB, just 3587 * send it. 3588 */ 3589 tcp_init_nondata_skb(skb, tp->snd_una - !urgent, TCPHDR_ACK); 3590 NET_INC_STATS(sock_net(sk), mib); 3591 return tcp_transmit_skb(sk, skb, 0, (__force gfp_t)0); 3592 } 3593 3594 /* Called from setsockopt( ... TCP_REPAIR ) */ 3595 void tcp_send_window_probe(struct sock *sk) 3596 { 3597 if (sk->sk_state == TCP_ESTABLISHED) { 3598 tcp_sk(sk)->snd_wl1 = tcp_sk(sk)->rcv_nxt - 1; 3599 tcp_mstamp_refresh(tcp_sk(sk)); 3600 tcp_xmit_probe_skb(sk, 0, LINUX_MIB_TCPWINPROBE); 3601 } 3602 } 3603 3604 /* Initiate keepalive or window probe from timer. */ 3605 int tcp_write_wakeup(struct sock *sk, int mib) 3606 { 3607 struct tcp_sock *tp = tcp_sk(sk); 3608 struct sk_buff *skb; 3609 3610 if (sk->sk_state == TCP_CLOSE) 3611 return -1; 3612 3613 skb = tcp_send_head(sk); 3614 if (skb && before(TCP_SKB_CB(skb)->seq, tcp_wnd_end(tp))) { 3615 int err; 3616 unsigned int mss = tcp_current_mss(sk); 3617 unsigned int seg_size = tcp_wnd_end(tp) - TCP_SKB_CB(skb)->seq; 3618 3619 if (before(tp->pushed_seq, TCP_SKB_CB(skb)->end_seq)) 3620 tp->pushed_seq = TCP_SKB_CB(skb)->end_seq; 3621 3622 /* We are probing the opening of a window 3623 * but the window size is != 0 3624 * must have been a result SWS avoidance ( sender ) 3625 */ 3626 if (seg_size < TCP_SKB_CB(skb)->end_seq - TCP_SKB_CB(skb)->seq || 3627 skb->len > mss) { 3628 seg_size = min(seg_size, mss); 3629 TCP_SKB_CB(skb)->tcp_flags |= TCPHDR_PSH; 3630 if (tcp_fragment(sk, skb, seg_size, mss, GFP_ATOMIC)) 3631 return -1; 3632 } else if (!tcp_skb_pcount(skb)) 3633 tcp_set_skb_tso_segs(skb, mss); 3634 3635 TCP_SKB_CB(skb)->tcp_flags |= TCPHDR_PSH; 3636 err = tcp_transmit_skb(sk, skb, 1, GFP_ATOMIC); 3637 if (!err) 3638 tcp_event_new_data_sent(sk, skb); 3639 return err; 3640 } else { 3641 if (between(tp->snd_up, tp->snd_una + 1, tp->snd_una + 0xFFFF)) 3642 tcp_xmit_probe_skb(sk, 1, mib); 3643 return tcp_xmit_probe_skb(sk, 0, mib); 3644 } 3645 } 3646 3647 /* A window probe timeout has occurred. If window is not closed send 3648 * a partial packet else a zero probe. 3649 */ 3650 void tcp_send_probe0(struct sock *sk) 3651 { 3652 struct inet_connection_sock *icsk = inet_csk(sk); 3653 struct tcp_sock *tp = tcp_sk(sk); 3654 struct net *net = sock_net(sk); 3655 unsigned long probe_max; 3656 int err; 3657 3658 err = tcp_write_wakeup(sk, LINUX_MIB_TCPWINPROBE); 3659 3660 if (tp->packets_out || !tcp_send_head(sk)) { 3661 /* Cancel probe timer, if it is not required. */ 3662 icsk->icsk_probes_out = 0; 3663 icsk->icsk_backoff = 0; 3664 return; 3665 } 3666 3667 if (err <= 0) { 3668 if (icsk->icsk_backoff < net->ipv4.sysctl_tcp_retries2) 3669 icsk->icsk_backoff++; 3670 icsk->icsk_probes_out++; 3671 probe_max = TCP_RTO_MAX; 3672 } else { 3673 /* If packet was not sent due to local congestion, 3674 * do not backoff and do not remember icsk_probes_out. 3675 * Let local senders to fight for local resources. 3676 * 3677 * Use accumulated backoff yet. 3678 */ 3679 if (!icsk->icsk_probes_out) 3680 icsk->icsk_probes_out = 1; 3681 probe_max = TCP_RESOURCE_PROBE_INTERVAL; 3682 } 3683 inet_csk_reset_xmit_timer(sk, ICSK_TIME_PROBE0, 3684 tcp_probe0_when(sk, probe_max), 3685 TCP_RTO_MAX); 3686 } 3687 3688 int tcp_rtx_synack(const struct sock *sk, struct request_sock *req) 3689 { 3690 const struct tcp_request_sock_ops *af_ops = tcp_rsk(req)->af_specific; 3691 struct flowi fl; 3692 int res; 3693 3694 tcp_rsk(req)->txhash = net_tx_rndhash(); 3695 res = af_ops->send_synack(sk, NULL, &fl, req, NULL, TCP_SYNACK_NORMAL); 3696 if (!res) { 3697 __TCP_INC_STATS(sock_net(sk), TCP_MIB_RETRANSSEGS); 3698 __NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPSYNRETRANS); 3699 if (unlikely(tcp_passive_fastopen(sk))) 3700 tcp_sk(sk)->total_retrans++; 3701 } 3702 return res; 3703 } 3704 EXPORT_SYMBOL(tcp_rtx_synack); 3705