xref: /openbmc/linux/net/ipv4/tcp_output.c (revision e9088629)
1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * INET		An implementation of the TCP/IP protocol suite for the LINUX
4  *		operating system.  INET is implemented using the  BSD Socket
5  *		interface as the means of communication with the user level.
6  *
7  *		Implementation of the Transmission Control Protocol(TCP).
8  *
9  * Authors:	Ross Biro
10  *		Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
11  *		Mark Evans, <evansmp@uhura.aston.ac.uk>
12  *		Corey Minyard <wf-rch!minyard@relay.EU.net>
13  *		Florian La Roche, <flla@stud.uni-sb.de>
14  *		Charles Hedrick, <hedrick@klinzhai.rutgers.edu>
15  *		Linus Torvalds, <torvalds@cs.helsinki.fi>
16  *		Alan Cox, <gw4pts@gw4pts.ampr.org>
17  *		Matthew Dillon, <dillon@apollo.west.oic.com>
18  *		Arnt Gulbrandsen, <agulbra@nvg.unit.no>
19  *		Jorge Cwik, <jorge@laser.satlink.net>
20  */
21 
22 /*
23  * Changes:	Pedro Roque	:	Retransmit queue handled by TCP.
24  *				:	Fragmentation on mtu decrease
25  *				:	Segment collapse on retransmit
26  *				:	AF independence
27  *
28  *		Linus Torvalds	:	send_delayed_ack
29  *		David S. Miller	:	Charge memory using the right skb
30  *					during syn/ack processing.
31  *		David S. Miller :	Output engine completely rewritten.
32  *		Andrea Arcangeli:	SYNACK carry ts_recent in tsecr.
33  *		Cacophonix Gaul :	draft-minshall-nagle-01
34  *		J Hadi Salim	:	ECN support
35  *
36  */
37 
38 #define pr_fmt(fmt) "TCP: " fmt
39 
40 #include <net/tcp.h>
41 #include <net/mptcp.h>
42 
43 #include <linux/compiler.h>
44 #include <linux/gfp.h>
45 #include <linux/module.h>
46 #include <linux/static_key.h>
47 
48 #include <trace/events/tcp.h>
49 
50 /* Refresh clocks of a TCP socket,
51  * ensuring monotically increasing values.
52  */
53 void tcp_mstamp_refresh(struct tcp_sock *tp)
54 {
55 	u64 val = tcp_clock_ns();
56 
57 	tp->tcp_clock_cache = val;
58 	tp->tcp_mstamp = div_u64(val, NSEC_PER_USEC);
59 }
60 
61 static bool tcp_write_xmit(struct sock *sk, unsigned int mss_now, int nonagle,
62 			   int push_one, gfp_t gfp);
63 
64 /* Account for new data that has been sent to the network. */
65 static void tcp_event_new_data_sent(struct sock *sk, struct sk_buff *skb)
66 {
67 	struct inet_connection_sock *icsk = inet_csk(sk);
68 	struct tcp_sock *tp = tcp_sk(sk);
69 	unsigned int prior_packets = tp->packets_out;
70 
71 	WRITE_ONCE(tp->snd_nxt, TCP_SKB_CB(skb)->end_seq);
72 
73 	__skb_unlink(skb, &sk->sk_write_queue);
74 	tcp_rbtree_insert(&sk->tcp_rtx_queue, skb);
75 
76 	if (tp->highest_sack == NULL)
77 		tp->highest_sack = skb;
78 
79 	tp->packets_out += tcp_skb_pcount(skb);
80 	if (!prior_packets || icsk->icsk_pending == ICSK_TIME_LOSS_PROBE)
81 		tcp_rearm_rto(sk);
82 
83 	NET_ADD_STATS(sock_net(sk), LINUX_MIB_TCPORIGDATASENT,
84 		      tcp_skb_pcount(skb));
85 	tcp_check_space(sk);
86 }
87 
88 /* SND.NXT, if window was not shrunk or the amount of shrunk was less than one
89  * window scaling factor due to loss of precision.
90  * If window has been shrunk, what should we make? It is not clear at all.
91  * Using SND.UNA we will fail to open window, SND.NXT is out of window. :-(
92  * Anything in between SND.UNA...SND.UNA+SND.WND also can be already
93  * invalid. OK, let's make this for now:
94  */
95 static inline __u32 tcp_acceptable_seq(const struct sock *sk)
96 {
97 	const struct tcp_sock *tp = tcp_sk(sk);
98 
99 	if (!before(tcp_wnd_end(tp), tp->snd_nxt) ||
100 	    (tp->rx_opt.wscale_ok &&
101 	     ((tp->snd_nxt - tcp_wnd_end(tp)) < (1 << tp->rx_opt.rcv_wscale))))
102 		return tp->snd_nxt;
103 	else
104 		return tcp_wnd_end(tp);
105 }
106 
107 /* Calculate mss to advertise in SYN segment.
108  * RFC1122, RFC1063, draft-ietf-tcpimpl-pmtud-01 state that:
109  *
110  * 1. It is independent of path mtu.
111  * 2. Ideally, it is maximal possible segment size i.e. 65535-40.
112  * 3. For IPv4 it is reasonable to calculate it from maximal MTU of
113  *    attached devices, because some buggy hosts are confused by
114  *    large MSS.
115  * 4. We do not make 3, we advertise MSS, calculated from first
116  *    hop device mtu, but allow to raise it to ip_rt_min_advmss.
117  *    This may be overridden via information stored in routing table.
118  * 5. Value 65535 for MSS is valid in IPv6 and means "as large as possible,
119  *    probably even Jumbo".
120  */
121 static __u16 tcp_advertise_mss(struct sock *sk)
122 {
123 	struct tcp_sock *tp = tcp_sk(sk);
124 	const struct dst_entry *dst = __sk_dst_get(sk);
125 	int mss = tp->advmss;
126 
127 	if (dst) {
128 		unsigned int metric = dst_metric_advmss(dst);
129 
130 		if (metric < mss) {
131 			mss = metric;
132 			tp->advmss = mss;
133 		}
134 	}
135 
136 	return (__u16)mss;
137 }
138 
139 /* RFC2861. Reset CWND after idle period longer RTO to "restart window".
140  * This is the first part of cwnd validation mechanism.
141  */
142 void tcp_cwnd_restart(struct sock *sk, s32 delta)
143 {
144 	struct tcp_sock *tp = tcp_sk(sk);
145 	u32 restart_cwnd = tcp_init_cwnd(tp, __sk_dst_get(sk));
146 	u32 cwnd = tcp_snd_cwnd(tp);
147 
148 	tcp_ca_event(sk, CA_EVENT_CWND_RESTART);
149 
150 	tp->snd_ssthresh = tcp_current_ssthresh(sk);
151 	restart_cwnd = min(restart_cwnd, cwnd);
152 
153 	while ((delta -= inet_csk(sk)->icsk_rto) > 0 && cwnd > restart_cwnd)
154 		cwnd >>= 1;
155 	tcp_snd_cwnd_set(tp, max(cwnd, restart_cwnd));
156 	tp->snd_cwnd_stamp = tcp_jiffies32;
157 	tp->snd_cwnd_used = 0;
158 }
159 
160 /* Congestion state accounting after a packet has been sent. */
161 static void tcp_event_data_sent(struct tcp_sock *tp,
162 				struct sock *sk)
163 {
164 	struct inet_connection_sock *icsk = inet_csk(sk);
165 	const u32 now = tcp_jiffies32;
166 
167 	if (tcp_packets_in_flight(tp) == 0)
168 		tcp_ca_event(sk, CA_EVENT_TX_START);
169 
170 	/* If this is the first data packet sent in response to the
171 	 * previous received data,
172 	 * and it is a reply for ato after last received packet,
173 	 * increase pingpong count.
174 	 */
175 	if (before(tp->lsndtime, icsk->icsk_ack.lrcvtime) &&
176 	    (u32)(now - icsk->icsk_ack.lrcvtime) < icsk->icsk_ack.ato)
177 		inet_csk_inc_pingpong_cnt(sk);
178 
179 	tp->lsndtime = now;
180 }
181 
182 /* Account for an ACK we sent. */
183 static inline void tcp_event_ack_sent(struct sock *sk, unsigned int pkts,
184 				      u32 rcv_nxt)
185 {
186 	struct tcp_sock *tp = tcp_sk(sk);
187 
188 	if (unlikely(tp->compressed_ack)) {
189 		NET_ADD_STATS(sock_net(sk), LINUX_MIB_TCPACKCOMPRESSED,
190 			      tp->compressed_ack);
191 		tp->compressed_ack = 0;
192 		if (hrtimer_try_to_cancel(&tp->compressed_ack_timer) == 1)
193 			__sock_put(sk);
194 	}
195 
196 	if (unlikely(rcv_nxt != tp->rcv_nxt))
197 		return;  /* Special ACK sent by DCTCP to reflect ECN */
198 	tcp_dec_quickack_mode(sk, pkts);
199 	inet_csk_clear_xmit_timer(sk, ICSK_TIME_DACK);
200 }
201 
202 /* Determine a window scaling and initial window to offer.
203  * Based on the assumption that the given amount of space
204  * will be offered. Store the results in the tp structure.
205  * NOTE: for smooth operation initial space offering should
206  * be a multiple of mss if possible. We assume here that mss >= 1.
207  * This MUST be enforced by all callers.
208  */
209 void tcp_select_initial_window(const struct sock *sk, int __space, __u32 mss,
210 			       __u32 *rcv_wnd, __u32 *window_clamp,
211 			       int wscale_ok, __u8 *rcv_wscale,
212 			       __u32 init_rcv_wnd)
213 {
214 	unsigned int space = (__space < 0 ? 0 : __space);
215 
216 	/* If no clamp set the clamp to the max possible scaled window */
217 	if (*window_clamp == 0)
218 		(*window_clamp) = (U16_MAX << TCP_MAX_WSCALE);
219 	space = min(*window_clamp, space);
220 
221 	/* Quantize space offering to a multiple of mss if possible. */
222 	if (space > mss)
223 		space = rounddown(space, mss);
224 
225 	/* NOTE: offering an initial window larger than 32767
226 	 * will break some buggy TCP stacks. If the admin tells us
227 	 * it is likely we could be speaking with such a buggy stack
228 	 * we will truncate our initial window offering to 32K-1
229 	 * unless the remote has sent us a window scaling option,
230 	 * which we interpret as a sign the remote TCP is not
231 	 * misinterpreting the window field as a signed quantity.
232 	 */
233 	if (sock_net(sk)->ipv4.sysctl_tcp_workaround_signed_windows)
234 		(*rcv_wnd) = min(space, MAX_TCP_WINDOW);
235 	else
236 		(*rcv_wnd) = min_t(u32, space, U16_MAX);
237 
238 	if (init_rcv_wnd)
239 		*rcv_wnd = min(*rcv_wnd, init_rcv_wnd * mss);
240 
241 	*rcv_wscale = 0;
242 	if (wscale_ok) {
243 		/* Set window scaling on max possible window */
244 		space = max_t(u32, space, sock_net(sk)->ipv4.sysctl_tcp_rmem[2]);
245 		space = max_t(u32, space, sysctl_rmem_max);
246 		space = min_t(u32, space, *window_clamp);
247 		*rcv_wscale = clamp_t(int, ilog2(space) - 15,
248 				      0, TCP_MAX_WSCALE);
249 	}
250 	/* Set the clamp no higher than max representable value */
251 	(*window_clamp) = min_t(__u32, U16_MAX << (*rcv_wscale), *window_clamp);
252 }
253 EXPORT_SYMBOL(tcp_select_initial_window);
254 
255 /* Chose a new window to advertise, update state in tcp_sock for the
256  * socket, and return result with RFC1323 scaling applied.  The return
257  * value can be stuffed directly into th->window for an outgoing
258  * frame.
259  */
260 static u16 tcp_select_window(struct sock *sk)
261 {
262 	struct tcp_sock *tp = tcp_sk(sk);
263 	u32 old_win = tp->rcv_wnd;
264 	u32 cur_win = tcp_receive_window(tp);
265 	u32 new_win = __tcp_select_window(sk);
266 
267 	/* Never shrink the offered window */
268 	if (new_win < cur_win) {
269 		/* Danger Will Robinson!
270 		 * Don't update rcv_wup/rcv_wnd here or else
271 		 * we will not be able to advertise a zero
272 		 * window in time.  --DaveM
273 		 *
274 		 * Relax Will Robinson.
275 		 */
276 		if (new_win == 0)
277 			NET_INC_STATS(sock_net(sk),
278 				      LINUX_MIB_TCPWANTZEROWINDOWADV);
279 		new_win = ALIGN(cur_win, 1 << tp->rx_opt.rcv_wscale);
280 	}
281 	tp->rcv_wnd = new_win;
282 	tp->rcv_wup = tp->rcv_nxt;
283 
284 	/* Make sure we do not exceed the maximum possible
285 	 * scaled window.
286 	 */
287 	if (!tp->rx_opt.rcv_wscale &&
288 	    sock_net(sk)->ipv4.sysctl_tcp_workaround_signed_windows)
289 		new_win = min(new_win, MAX_TCP_WINDOW);
290 	else
291 		new_win = min(new_win, (65535U << tp->rx_opt.rcv_wscale));
292 
293 	/* RFC1323 scaling applied */
294 	new_win >>= tp->rx_opt.rcv_wscale;
295 
296 	/* If we advertise zero window, disable fast path. */
297 	if (new_win == 0) {
298 		tp->pred_flags = 0;
299 		if (old_win)
300 			NET_INC_STATS(sock_net(sk),
301 				      LINUX_MIB_TCPTOZEROWINDOWADV);
302 	} else if (old_win == 0) {
303 		NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPFROMZEROWINDOWADV);
304 	}
305 
306 	return new_win;
307 }
308 
309 /* Packet ECN state for a SYN-ACK */
310 static void tcp_ecn_send_synack(struct sock *sk, struct sk_buff *skb)
311 {
312 	const struct tcp_sock *tp = tcp_sk(sk);
313 
314 	TCP_SKB_CB(skb)->tcp_flags &= ~TCPHDR_CWR;
315 	if (!(tp->ecn_flags & TCP_ECN_OK))
316 		TCP_SKB_CB(skb)->tcp_flags &= ~TCPHDR_ECE;
317 	else if (tcp_ca_needs_ecn(sk) ||
318 		 tcp_bpf_ca_needs_ecn(sk))
319 		INET_ECN_xmit(sk);
320 }
321 
322 /* Packet ECN state for a SYN.  */
323 static void tcp_ecn_send_syn(struct sock *sk, struct sk_buff *skb)
324 {
325 	struct tcp_sock *tp = tcp_sk(sk);
326 	bool bpf_needs_ecn = tcp_bpf_ca_needs_ecn(sk);
327 	bool use_ecn = READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_ecn) == 1 ||
328 		tcp_ca_needs_ecn(sk) || bpf_needs_ecn;
329 
330 	if (!use_ecn) {
331 		const struct dst_entry *dst = __sk_dst_get(sk);
332 
333 		if (dst && dst_feature(dst, RTAX_FEATURE_ECN))
334 			use_ecn = true;
335 	}
336 
337 	tp->ecn_flags = 0;
338 
339 	if (use_ecn) {
340 		TCP_SKB_CB(skb)->tcp_flags |= TCPHDR_ECE | TCPHDR_CWR;
341 		tp->ecn_flags = TCP_ECN_OK;
342 		if (tcp_ca_needs_ecn(sk) || bpf_needs_ecn)
343 			INET_ECN_xmit(sk);
344 	}
345 }
346 
347 static void tcp_ecn_clear_syn(struct sock *sk, struct sk_buff *skb)
348 {
349 	if (READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_ecn_fallback))
350 		/* tp->ecn_flags are cleared at a later point in time when
351 		 * SYN ACK is ultimatively being received.
352 		 */
353 		TCP_SKB_CB(skb)->tcp_flags &= ~(TCPHDR_ECE | TCPHDR_CWR);
354 }
355 
356 static void
357 tcp_ecn_make_synack(const struct request_sock *req, struct tcphdr *th)
358 {
359 	if (inet_rsk(req)->ecn_ok)
360 		th->ece = 1;
361 }
362 
363 /* Set up ECN state for a packet on a ESTABLISHED socket that is about to
364  * be sent.
365  */
366 static void tcp_ecn_send(struct sock *sk, struct sk_buff *skb,
367 			 struct tcphdr *th, int tcp_header_len)
368 {
369 	struct tcp_sock *tp = tcp_sk(sk);
370 
371 	if (tp->ecn_flags & TCP_ECN_OK) {
372 		/* Not-retransmitted data segment: set ECT and inject CWR. */
373 		if (skb->len != tcp_header_len &&
374 		    !before(TCP_SKB_CB(skb)->seq, tp->snd_nxt)) {
375 			INET_ECN_xmit(sk);
376 			if (tp->ecn_flags & TCP_ECN_QUEUE_CWR) {
377 				tp->ecn_flags &= ~TCP_ECN_QUEUE_CWR;
378 				th->cwr = 1;
379 				skb_shinfo(skb)->gso_type |= SKB_GSO_TCP_ECN;
380 			}
381 		} else if (!tcp_ca_needs_ecn(sk)) {
382 			/* ACK or retransmitted segment: clear ECT|CE */
383 			INET_ECN_dontxmit(sk);
384 		}
385 		if (tp->ecn_flags & TCP_ECN_DEMAND_CWR)
386 			th->ece = 1;
387 	}
388 }
389 
390 /* Constructs common control bits of non-data skb. If SYN/FIN is present,
391  * auto increment end seqno.
392  */
393 static void tcp_init_nondata_skb(struct sk_buff *skb, u32 seq, u8 flags)
394 {
395 	skb->ip_summed = CHECKSUM_PARTIAL;
396 
397 	TCP_SKB_CB(skb)->tcp_flags = flags;
398 
399 	tcp_skb_pcount_set(skb, 1);
400 
401 	TCP_SKB_CB(skb)->seq = seq;
402 	if (flags & (TCPHDR_SYN | TCPHDR_FIN))
403 		seq++;
404 	TCP_SKB_CB(skb)->end_seq = seq;
405 }
406 
407 static inline bool tcp_urg_mode(const struct tcp_sock *tp)
408 {
409 	return tp->snd_una != tp->snd_up;
410 }
411 
412 #define OPTION_SACK_ADVERTISE	BIT(0)
413 #define OPTION_TS		BIT(1)
414 #define OPTION_MD5		BIT(2)
415 #define OPTION_WSCALE		BIT(3)
416 #define OPTION_FAST_OPEN_COOKIE	BIT(8)
417 #define OPTION_SMC		BIT(9)
418 #define OPTION_MPTCP		BIT(10)
419 
420 static void smc_options_write(__be32 *ptr, u16 *options)
421 {
422 #if IS_ENABLED(CONFIG_SMC)
423 	if (static_branch_unlikely(&tcp_have_smc)) {
424 		if (unlikely(OPTION_SMC & *options)) {
425 			*ptr++ = htonl((TCPOPT_NOP  << 24) |
426 				       (TCPOPT_NOP  << 16) |
427 				       (TCPOPT_EXP <<  8) |
428 				       (TCPOLEN_EXP_SMC_BASE));
429 			*ptr++ = htonl(TCPOPT_SMC_MAGIC);
430 		}
431 	}
432 #endif
433 }
434 
435 struct tcp_out_options {
436 	u16 options;		/* bit field of OPTION_* */
437 	u16 mss;		/* 0 to disable */
438 	u8 ws;			/* window scale, 0 to disable */
439 	u8 num_sack_blocks;	/* number of SACK blocks to include */
440 	u8 hash_size;		/* bytes in hash_location */
441 	u8 bpf_opt_len;		/* length of BPF hdr option */
442 	__u8 *hash_location;	/* temporary pointer, overloaded */
443 	__u32 tsval, tsecr;	/* need to include OPTION_TS */
444 	struct tcp_fastopen_cookie *fastopen_cookie;	/* Fast open cookie */
445 	struct mptcp_out_options mptcp;
446 };
447 
448 static void mptcp_options_write(struct tcphdr *th, __be32 *ptr,
449 				struct tcp_sock *tp,
450 				struct tcp_out_options *opts)
451 {
452 #if IS_ENABLED(CONFIG_MPTCP)
453 	if (unlikely(OPTION_MPTCP & opts->options))
454 		mptcp_write_options(th, ptr, tp, &opts->mptcp);
455 #endif
456 }
457 
458 #ifdef CONFIG_CGROUP_BPF
459 static int bpf_skops_write_hdr_opt_arg0(struct sk_buff *skb,
460 					enum tcp_synack_type synack_type)
461 {
462 	if (unlikely(!skb))
463 		return BPF_WRITE_HDR_TCP_CURRENT_MSS;
464 
465 	if (unlikely(synack_type == TCP_SYNACK_COOKIE))
466 		return BPF_WRITE_HDR_TCP_SYNACK_COOKIE;
467 
468 	return 0;
469 }
470 
471 /* req, syn_skb and synack_type are used when writing synack */
472 static void bpf_skops_hdr_opt_len(struct sock *sk, struct sk_buff *skb,
473 				  struct request_sock *req,
474 				  struct sk_buff *syn_skb,
475 				  enum tcp_synack_type synack_type,
476 				  struct tcp_out_options *opts,
477 				  unsigned int *remaining)
478 {
479 	struct bpf_sock_ops_kern sock_ops;
480 	int err;
481 
482 	if (likely(!BPF_SOCK_OPS_TEST_FLAG(tcp_sk(sk),
483 					   BPF_SOCK_OPS_WRITE_HDR_OPT_CB_FLAG)) ||
484 	    !*remaining)
485 		return;
486 
487 	/* *remaining has already been aligned to 4 bytes, so *remaining >= 4 */
488 
489 	/* init sock_ops */
490 	memset(&sock_ops, 0, offsetof(struct bpf_sock_ops_kern, temp));
491 
492 	sock_ops.op = BPF_SOCK_OPS_HDR_OPT_LEN_CB;
493 
494 	if (req) {
495 		/* The listen "sk" cannot be passed here because
496 		 * it is not locked.  It would not make too much
497 		 * sense to do bpf_setsockopt(listen_sk) based
498 		 * on individual connection request also.
499 		 *
500 		 * Thus, "req" is passed here and the cgroup-bpf-progs
501 		 * of the listen "sk" will be run.
502 		 *
503 		 * "req" is also used here for fastopen even the "sk" here is
504 		 * a fullsock "child" sk.  It is to keep the behavior
505 		 * consistent between fastopen and non-fastopen on
506 		 * the bpf programming side.
507 		 */
508 		sock_ops.sk = (struct sock *)req;
509 		sock_ops.syn_skb = syn_skb;
510 	} else {
511 		sock_owned_by_me(sk);
512 
513 		sock_ops.is_fullsock = 1;
514 		sock_ops.sk = sk;
515 	}
516 
517 	sock_ops.args[0] = bpf_skops_write_hdr_opt_arg0(skb, synack_type);
518 	sock_ops.remaining_opt_len = *remaining;
519 	/* tcp_current_mss() does not pass a skb */
520 	if (skb)
521 		bpf_skops_init_skb(&sock_ops, skb, 0);
522 
523 	err = BPF_CGROUP_RUN_PROG_SOCK_OPS_SK(&sock_ops, sk);
524 
525 	if (err || sock_ops.remaining_opt_len == *remaining)
526 		return;
527 
528 	opts->bpf_opt_len = *remaining - sock_ops.remaining_opt_len;
529 	/* round up to 4 bytes */
530 	opts->bpf_opt_len = (opts->bpf_opt_len + 3) & ~3;
531 
532 	*remaining -= opts->bpf_opt_len;
533 }
534 
535 static void bpf_skops_write_hdr_opt(struct sock *sk, struct sk_buff *skb,
536 				    struct request_sock *req,
537 				    struct sk_buff *syn_skb,
538 				    enum tcp_synack_type synack_type,
539 				    struct tcp_out_options *opts)
540 {
541 	u8 first_opt_off, nr_written, max_opt_len = opts->bpf_opt_len;
542 	struct bpf_sock_ops_kern sock_ops;
543 	int err;
544 
545 	if (likely(!max_opt_len))
546 		return;
547 
548 	memset(&sock_ops, 0, offsetof(struct bpf_sock_ops_kern, temp));
549 
550 	sock_ops.op = BPF_SOCK_OPS_WRITE_HDR_OPT_CB;
551 
552 	if (req) {
553 		sock_ops.sk = (struct sock *)req;
554 		sock_ops.syn_skb = syn_skb;
555 	} else {
556 		sock_owned_by_me(sk);
557 
558 		sock_ops.is_fullsock = 1;
559 		sock_ops.sk = sk;
560 	}
561 
562 	sock_ops.args[0] = bpf_skops_write_hdr_opt_arg0(skb, synack_type);
563 	sock_ops.remaining_opt_len = max_opt_len;
564 	first_opt_off = tcp_hdrlen(skb) - max_opt_len;
565 	bpf_skops_init_skb(&sock_ops, skb, first_opt_off);
566 
567 	err = BPF_CGROUP_RUN_PROG_SOCK_OPS_SK(&sock_ops, sk);
568 
569 	if (err)
570 		nr_written = 0;
571 	else
572 		nr_written = max_opt_len - sock_ops.remaining_opt_len;
573 
574 	if (nr_written < max_opt_len)
575 		memset(skb->data + first_opt_off + nr_written, TCPOPT_NOP,
576 		       max_opt_len - nr_written);
577 }
578 #else
579 static void bpf_skops_hdr_opt_len(struct sock *sk, struct sk_buff *skb,
580 				  struct request_sock *req,
581 				  struct sk_buff *syn_skb,
582 				  enum tcp_synack_type synack_type,
583 				  struct tcp_out_options *opts,
584 				  unsigned int *remaining)
585 {
586 }
587 
588 static void bpf_skops_write_hdr_opt(struct sock *sk, struct sk_buff *skb,
589 				    struct request_sock *req,
590 				    struct sk_buff *syn_skb,
591 				    enum tcp_synack_type synack_type,
592 				    struct tcp_out_options *opts)
593 {
594 }
595 #endif
596 
597 /* Write previously computed TCP options to the packet.
598  *
599  * Beware: Something in the Internet is very sensitive to the ordering of
600  * TCP options, we learned this through the hard way, so be careful here.
601  * Luckily we can at least blame others for their non-compliance but from
602  * inter-operability perspective it seems that we're somewhat stuck with
603  * the ordering which we have been using if we want to keep working with
604  * those broken things (not that it currently hurts anybody as there isn't
605  * particular reason why the ordering would need to be changed).
606  *
607  * At least SACK_PERM as the first option is known to lead to a disaster
608  * (but it may well be that other scenarios fail similarly).
609  */
610 static void tcp_options_write(struct tcphdr *th, struct tcp_sock *tp,
611 			      struct tcp_out_options *opts)
612 {
613 	__be32 *ptr = (__be32 *)(th + 1);
614 	u16 options = opts->options;	/* mungable copy */
615 
616 	if (unlikely(OPTION_MD5 & options)) {
617 		*ptr++ = htonl((TCPOPT_NOP << 24) | (TCPOPT_NOP << 16) |
618 			       (TCPOPT_MD5SIG << 8) | TCPOLEN_MD5SIG);
619 		/* overload cookie hash location */
620 		opts->hash_location = (__u8 *)ptr;
621 		ptr += 4;
622 	}
623 
624 	if (unlikely(opts->mss)) {
625 		*ptr++ = htonl((TCPOPT_MSS << 24) |
626 			       (TCPOLEN_MSS << 16) |
627 			       opts->mss);
628 	}
629 
630 	if (likely(OPTION_TS & options)) {
631 		if (unlikely(OPTION_SACK_ADVERTISE & options)) {
632 			*ptr++ = htonl((TCPOPT_SACK_PERM << 24) |
633 				       (TCPOLEN_SACK_PERM << 16) |
634 				       (TCPOPT_TIMESTAMP << 8) |
635 				       TCPOLEN_TIMESTAMP);
636 			options &= ~OPTION_SACK_ADVERTISE;
637 		} else {
638 			*ptr++ = htonl((TCPOPT_NOP << 24) |
639 				       (TCPOPT_NOP << 16) |
640 				       (TCPOPT_TIMESTAMP << 8) |
641 				       TCPOLEN_TIMESTAMP);
642 		}
643 		*ptr++ = htonl(opts->tsval);
644 		*ptr++ = htonl(opts->tsecr);
645 	}
646 
647 	if (unlikely(OPTION_SACK_ADVERTISE & options)) {
648 		*ptr++ = htonl((TCPOPT_NOP << 24) |
649 			       (TCPOPT_NOP << 16) |
650 			       (TCPOPT_SACK_PERM << 8) |
651 			       TCPOLEN_SACK_PERM);
652 	}
653 
654 	if (unlikely(OPTION_WSCALE & options)) {
655 		*ptr++ = htonl((TCPOPT_NOP << 24) |
656 			       (TCPOPT_WINDOW << 16) |
657 			       (TCPOLEN_WINDOW << 8) |
658 			       opts->ws);
659 	}
660 
661 	if (unlikely(opts->num_sack_blocks)) {
662 		struct tcp_sack_block *sp = tp->rx_opt.dsack ?
663 			tp->duplicate_sack : tp->selective_acks;
664 		int this_sack;
665 
666 		*ptr++ = htonl((TCPOPT_NOP  << 24) |
667 			       (TCPOPT_NOP  << 16) |
668 			       (TCPOPT_SACK <<  8) |
669 			       (TCPOLEN_SACK_BASE + (opts->num_sack_blocks *
670 						     TCPOLEN_SACK_PERBLOCK)));
671 
672 		for (this_sack = 0; this_sack < opts->num_sack_blocks;
673 		     ++this_sack) {
674 			*ptr++ = htonl(sp[this_sack].start_seq);
675 			*ptr++ = htonl(sp[this_sack].end_seq);
676 		}
677 
678 		tp->rx_opt.dsack = 0;
679 	}
680 
681 	if (unlikely(OPTION_FAST_OPEN_COOKIE & options)) {
682 		struct tcp_fastopen_cookie *foc = opts->fastopen_cookie;
683 		u8 *p = (u8 *)ptr;
684 		u32 len; /* Fast Open option length */
685 
686 		if (foc->exp) {
687 			len = TCPOLEN_EXP_FASTOPEN_BASE + foc->len;
688 			*ptr = htonl((TCPOPT_EXP << 24) | (len << 16) |
689 				     TCPOPT_FASTOPEN_MAGIC);
690 			p += TCPOLEN_EXP_FASTOPEN_BASE;
691 		} else {
692 			len = TCPOLEN_FASTOPEN_BASE + foc->len;
693 			*p++ = TCPOPT_FASTOPEN;
694 			*p++ = len;
695 		}
696 
697 		memcpy(p, foc->val, foc->len);
698 		if ((len & 3) == 2) {
699 			p[foc->len] = TCPOPT_NOP;
700 			p[foc->len + 1] = TCPOPT_NOP;
701 		}
702 		ptr += (len + 3) >> 2;
703 	}
704 
705 	smc_options_write(ptr, &options);
706 
707 	mptcp_options_write(th, ptr, tp, opts);
708 }
709 
710 static void smc_set_option(const struct tcp_sock *tp,
711 			   struct tcp_out_options *opts,
712 			   unsigned int *remaining)
713 {
714 #if IS_ENABLED(CONFIG_SMC)
715 	if (static_branch_unlikely(&tcp_have_smc)) {
716 		if (tp->syn_smc) {
717 			if (*remaining >= TCPOLEN_EXP_SMC_BASE_ALIGNED) {
718 				opts->options |= OPTION_SMC;
719 				*remaining -= TCPOLEN_EXP_SMC_BASE_ALIGNED;
720 			}
721 		}
722 	}
723 #endif
724 }
725 
726 static void smc_set_option_cond(const struct tcp_sock *tp,
727 				const struct inet_request_sock *ireq,
728 				struct tcp_out_options *opts,
729 				unsigned int *remaining)
730 {
731 #if IS_ENABLED(CONFIG_SMC)
732 	if (static_branch_unlikely(&tcp_have_smc)) {
733 		if (tp->syn_smc && ireq->smc_ok) {
734 			if (*remaining >= TCPOLEN_EXP_SMC_BASE_ALIGNED) {
735 				opts->options |= OPTION_SMC;
736 				*remaining -= TCPOLEN_EXP_SMC_BASE_ALIGNED;
737 			}
738 		}
739 	}
740 #endif
741 }
742 
743 static void mptcp_set_option_cond(const struct request_sock *req,
744 				  struct tcp_out_options *opts,
745 				  unsigned int *remaining)
746 {
747 	if (rsk_is_mptcp(req)) {
748 		unsigned int size;
749 
750 		if (mptcp_synack_options(req, &size, &opts->mptcp)) {
751 			if (*remaining >= size) {
752 				opts->options |= OPTION_MPTCP;
753 				*remaining -= size;
754 			}
755 		}
756 	}
757 }
758 
759 /* Compute TCP options for SYN packets. This is not the final
760  * network wire format yet.
761  */
762 static unsigned int tcp_syn_options(struct sock *sk, struct sk_buff *skb,
763 				struct tcp_out_options *opts,
764 				struct tcp_md5sig_key **md5)
765 {
766 	struct tcp_sock *tp = tcp_sk(sk);
767 	unsigned int remaining = MAX_TCP_OPTION_SPACE;
768 	struct tcp_fastopen_request *fastopen = tp->fastopen_req;
769 
770 	*md5 = NULL;
771 #ifdef CONFIG_TCP_MD5SIG
772 	if (static_branch_unlikely(&tcp_md5_needed) &&
773 	    rcu_access_pointer(tp->md5sig_info)) {
774 		*md5 = tp->af_specific->md5_lookup(sk, sk);
775 		if (*md5) {
776 			opts->options |= OPTION_MD5;
777 			remaining -= TCPOLEN_MD5SIG_ALIGNED;
778 		}
779 	}
780 #endif
781 
782 	/* We always get an MSS option.  The option bytes which will be seen in
783 	 * normal data packets should timestamps be used, must be in the MSS
784 	 * advertised.  But we subtract them from tp->mss_cache so that
785 	 * calculations in tcp_sendmsg are simpler etc.  So account for this
786 	 * fact here if necessary.  If we don't do this correctly, as a
787 	 * receiver we won't recognize data packets as being full sized when we
788 	 * should, and thus we won't abide by the delayed ACK rules correctly.
789 	 * SACKs don't matter, we never delay an ACK when we have any of those
790 	 * going out.  */
791 	opts->mss = tcp_advertise_mss(sk);
792 	remaining -= TCPOLEN_MSS_ALIGNED;
793 
794 	if (likely(READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_timestamps) && !*md5)) {
795 		opts->options |= OPTION_TS;
796 		opts->tsval = tcp_skb_timestamp(skb) + tp->tsoffset;
797 		opts->tsecr = tp->rx_opt.ts_recent;
798 		remaining -= TCPOLEN_TSTAMP_ALIGNED;
799 	}
800 	if (likely(READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_window_scaling))) {
801 		opts->ws = tp->rx_opt.rcv_wscale;
802 		opts->options |= OPTION_WSCALE;
803 		remaining -= TCPOLEN_WSCALE_ALIGNED;
804 	}
805 	if (likely(READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_sack))) {
806 		opts->options |= OPTION_SACK_ADVERTISE;
807 		if (unlikely(!(OPTION_TS & opts->options)))
808 			remaining -= TCPOLEN_SACKPERM_ALIGNED;
809 	}
810 
811 	if (fastopen && fastopen->cookie.len >= 0) {
812 		u32 need = fastopen->cookie.len;
813 
814 		need += fastopen->cookie.exp ? TCPOLEN_EXP_FASTOPEN_BASE :
815 					       TCPOLEN_FASTOPEN_BASE;
816 		need = (need + 3) & ~3U;  /* Align to 32 bits */
817 		if (remaining >= need) {
818 			opts->options |= OPTION_FAST_OPEN_COOKIE;
819 			opts->fastopen_cookie = &fastopen->cookie;
820 			remaining -= need;
821 			tp->syn_fastopen = 1;
822 			tp->syn_fastopen_exp = fastopen->cookie.exp ? 1 : 0;
823 		}
824 	}
825 
826 	smc_set_option(tp, opts, &remaining);
827 
828 	if (sk_is_mptcp(sk)) {
829 		unsigned int size;
830 
831 		if (mptcp_syn_options(sk, skb, &size, &opts->mptcp)) {
832 			opts->options |= OPTION_MPTCP;
833 			remaining -= size;
834 		}
835 	}
836 
837 	bpf_skops_hdr_opt_len(sk, skb, NULL, NULL, 0, opts, &remaining);
838 
839 	return MAX_TCP_OPTION_SPACE - remaining;
840 }
841 
842 /* Set up TCP options for SYN-ACKs. */
843 static unsigned int tcp_synack_options(const struct sock *sk,
844 				       struct request_sock *req,
845 				       unsigned int mss, struct sk_buff *skb,
846 				       struct tcp_out_options *opts,
847 				       const struct tcp_md5sig_key *md5,
848 				       struct tcp_fastopen_cookie *foc,
849 				       enum tcp_synack_type synack_type,
850 				       struct sk_buff *syn_skb)
851 {
852 	struct inet_request_sock *ireq = inet_rsk(req);
853 	unsigned int remaining = MAX_TCP_OPTION_SPACE;
854 
855 #ifdef CONFIG_TCP_MD5SIG
856 	if (md5) {
857 		opts->options |= OPTION_MD5;
858 		remaining -= TCPOLEN_MD5SIG_ALIGNED;
859 
860 		/* We can't fit any SACK blocks in a packet with MD5 + TS
861 		 * options. There was discussion about disabling SACK
862 		 * rather than TS in order to fit in better with old,
863 		 * buggy kernels, but that was deemed to be unnecessary.
864 		 */
865 		if (synack_type != TCP_SYNACK_COOKIE)
866 			ireq->tstamp_ok &= !ireq->sack_ok;
867 	}
868 #endif
869 
870 	/* We always send an MSS option. */
871 	opts->mss = mss;
872 	remaining -= TCPOLEN_MSS_ALIGNED;
873 
874 	if (likely(ireq->wscale_ok)) {
875 		opts->ws = ireq->rcv_wscale;
876 		opts->options |= OPTION_WSCALE;
877 		remaining -= TCPOLEN_WSCALE_ALIGNED;
878 	}
879 	if (likely(ireq->tstamp_ok)) {
880 		opts->options |= OPTION_TS;
881 		opts->tsval = tcp_skb_timestamp(skb) + tcp_rsk(req)->ts_off;
882 		opts->tsecr = req->ts_recent;
883 		remaining -= TCPOLEN_TSTAMP_ALIGNED;
884 	}
885 	if (likely(ireq->sack_ok)) {
886 		opts->options |= OPTION_SACK_ADVERTISE;
887 		if (unlikely(!ireq->tstamp_ok))
888 			remaining -= TCPOLEN_SACKPERM_ALIGNED;
889 	}
890 	if (foc != NULL && foc->len >= 0) {
891 		u32 need = foc->len;
892 
893 		need += foc->exp ? TCPOLEN_EXP_FASTOPEN_BASE :
894 				   TCPOLEN_FASTOPEN_BASE;
895 		need = (need + 3) & ~3U;  /* Align to 32 bits */
896 		if (remaining >= need) {
897 			opts->options |= OPTION_FAST_OPEN_COOKIE;
898 			opts->fastopen_cookie = foc;
899 			remaining -= need;
900 		}
901 	}
902 
903 	mptcp_set_option_cond(req, opts, &remaining);
904 
905 	smc_set_option_cond(tcp_sk(sk), ireq, opts, &remaining);
906 
907 	bpf_skops_hdr_opt_len((struct sock *)sk, skb, req, syn_skb,
908 			      synack_type, opts, &remaining);
909 
910 	return MAX_TCP_OPTION_SPACE - remaining;
911 }
912 
913 /* Compute TCP options for ESTABLISHED sockets. This is not the
914  * final wire format yet.
915  */
916 static unsigned int tcp_established_options(struct sock *sk, struct sk_buff *skb,
917 					struct tcp_out_options *opts,
918 					struct tcp_md5sig_key **md5)
919 {
920 	struct tcp_sock *tp = tcp_sk(sk);
921 	unsigned int size = 0;
922 	unsigned int eff_sacks;
923 
924 	opts->options = 0;
925 
926 	*md5 = NULL;
927 #ifdef CONFIG_TCP_MD5SIG
928 	if (static_branch_unlikely(&tcp_md5_needed) &&
929 	    rcu_access_pointer(tp->md5sig_info)) {
930 		*md5 = tp->af_specific->md5_lookup(sk, sk);
931 		if (*md5) {
932 			opts->options |= OPTION_MD5;
933 			size += TCPOLEN_MD5SIG_ALIGNED;
934 		}
935 	}
936 #endif
937 
938 	if (likely(tp->rx_opt.tstamp_ok)) {
939 		opts->options |= OPTION_TS;
940 		opts->tsval = skb ? tcp_skb_timestamp(skb) + tp->tsoffset : 0;
941 		opts->tsecr = tp->rx_opt.ts_recent;
942 		size += TCPOLEN_TSTAMP_ALIGNED;
943 	}
944 
945 	/* MPTCP options have precedence over SACK for the limited TCP
946 	 * option space because a MPTCP connection would be forced to
947 	 * fall back to regular TCP if a required multipath option is
948 	 * missing. SACK still gets a chance to use whatever space is
949 	 * left.
950 	 */
951 	if (sk_is_mptcp(sk)) {
952 		unsigned int remaining = MAX_TCP_OPTION_SPACE - size;
953 		unsigned int opt_size = 0;
954 
955 		if (mptcp_established_options(sk, skb, &opt_size, remaining,
956 					      &opts->mptcp)) {
957 			opts->options |= OPTION_MPTCP;
958 			size += opt_size;
959 		}
960 	}
961 
962 	eff_sacks = tp->rx_opt.num_sacks + tp->rx_opt.dsack;
963 	if (unlikely(eff_sacks)) {
964 		const unsigned int remaining = MAX_TCP_OPTION_SPACE - size;
965 		if (unlikely(remaining < TCPOLEN_SACK_BASE_ALIGNED +
966 					 TCPOLEN_SACK_PERBLOCK))
967 			return size;
968 
969 		opts->num_sack_blocks =
970 			min_t(unsigned int, eff_sacks,
971 			      (remaining - TCPOLEN_SACK_BASE_ALIGNED) /
972 			      TCPOLEN_SACK_PERBLOCK);
973 
974 		size += TCPOLEN_SACK_BASE_ALIGNED +
975 			opts->num_sack_blocks * TCPOLEN_SACK_PERBLOCK;
976 	}
977 
978 	if (unlikely(BPF_SOCK_OPS_TEST_FLAG(tp,
979 					    BPF_SOCK_OPS_WRITE_HDR_OPT_CB_FLAG))) {
980 		unsigned int remaining = MAX_TCP_OPTION_SPACE - size;
981 
982 		bpf_skops_hdr_opt_len(sk, skb, NULL, NULL, 0, opts, &remaining);
983 
984 		size = MAX_TCP_OPTION_SPACE - remaining;
985 	}
986 
987 	return size;
988 }
989 
990 
991 /* TCP SMALL QUEUES (TSQ)
992  *
993  * TSQ goal is to keep small amount of skbs per tcp flow in tx queues (qdisc+dev)
994  * to reduce RTT and bufferbloat.
995  * We do this using a special skb destructor (tcp_wfree).
996  *
997  * Its important tcp_wfree() can be replaced by sock_wfree() in the event skb
998  * needs to be reallocated in a driver.
999  * The invariant being skb->truesize subtracted from sk->sk_wmem_alloc
1000  *
1001  * Since transmit from skb destructor is forbidden, we use a tasklet
1002  * to process all sockets that eventually need to send more skbs.
1003  * We use one tasklet per cpu, with its own queue of sockets.
1004  */
1005 struct tsq_tasklet {
1006 	struct tasklet_struct	tasklet;
1007 	struct list_head	head; /* queue of tcp sockets */
1008 };
1009 static DEFINE_PER_CPU(struct tsq_tasklet, tsq_tasklet);
1010 
1011 static void tcp_tsq_write(struct sock *sk)
1012 {
1013 	if ((1 << sk->sk_state) &
1014 	    (TCPF_ESTABLISHED | TCPF_FIN_WAIT1 | TCPF_CLOSING |
1015 	     TCPF_CLOSE_WAIT  | TCPF_LAST_ACK)) {
1016 		struct tcp_sock *tp = tcp_sk(sk);
1017 
1018 		if (tp->lost_out > tp->retrans_out &&
1019 		    tcp_snd_cwnd(tp) > tcp_packets_in_flight(tp)) {
1020 			tcp_mstamp_refresh(tp);
1021 			tcp_xmit_retransmit_queue(sk);
1022 		}
1023 
1024 		tcp_write_xmit(sk, tcp_current_mss(sk), tp->nonagle,
1025 			       0, GFP_ATOMIC);
1026 	}
1027 }
1028 
1029 static void tcp_tsq_handler(struct sock *sk)
1030 {
1031 	bh_lock_sock(sk);
1032 	if (!sock_owned_by_user(sk))
1033 		tcp_tsq_write(sk);
1034 	else if (!test_and_set_bit(TCP_TSQ_DEFERRED, &sk->sk_tsq_flags))
1035 		sock_hold(sk);
1036 	bh_unlock_sock(sk);
1037 }
1038 /*
1039  * One tasklet per cpu tries to send more skbs.
1040  * We run in tasklet context but need to disable irqs when
1041  * transferring tsq->head because tcp_wfree() might
1042  * interrupt us (non NAPI drivers)
1043  */
1044 static void tcp_tasklet_func(struct tasklet_struct *t)
1045 {
1046 	struct tsq_tasklet *tsq = from_tasklet(tsq,  t, tasklet);
1047 	LIST_HEAD(list);
1048 	unsigned long flags;
1049 	struct list_head *q, *n;
1050 	struct tcp_sock *tp;
1051 	struct sock *sk;
1052 
1053 	local_irq_save(flags);
1054 	list_splice_init(&tsq->head, &list);
1055 	local_irq_restore(flags);
1056 
1057 	list_for_each_safe(q, n, &list) {
1058 		tp = list_entry(q, struct tcp_sock, tsq_node);
1059 		list_del(&tp->tsq_node);
1060 
1061 		sk = (struct sock *)tp;
1062 		smp_mb__before_atomic();
1063 		clear_bit(TSQ_QUEUED, &sk->sk_tsq_flags);
1064 
1065 		tcp_tsq_handler(sk);
1066 		sk_free(sk);
1067 	}
1068 }
1069 
1070 #define TCP_DEFERRED_ALL (TCPF_TSQ_DEFERRED |		\
1071 			  TCPF_WRITE_TIMER_DEFERRED |	\
1072 			  TCPF_DELACK_TIMER_DEFERRED |	\
1073 			  TCPF_MTU_REDUCED_DEFERRED)
1074 /**
1075  * tcp_release_cb - tcp release_sock() callback
1076  * @sk: socket
1077  *
1078  * called from release_sock() to perform protocol dependent
1079  * actions before socket release.
1080  */
1081 void tcp_release_cb(struct sock *sk)
1082 {
1083 	unsigned long flags, nflags;
1084 
1085 	/* perform an atomic operation only if at least one flag is set */
1086 	do {
1087 		flags = sk->sk_tsq_flags;
1088 		if (!(flags & TCP_DEFERRED_ALL))
1089 			return;
1090 		nflags = flags & ~TCP_DEFERRED_ALL;
1091 	} while (cmpxchg(&sk->sk_tsq_flags, flags, nflags) != flags);
1092 
1093 	if (flags & TCPF_TSQ_DEFERRED) {
1094 		tcp_tsq_write(sk);
1095 		__sock_put(sk);
1096 	}
1097 	/* Here begins the tricky part :
1098 	 * We are called from release_sock() with :
1099 	 * 1) BH disabled
1100 	 * 2) sk_lock.slock spinlock held
1101 	 * 3) socket owned by us (sk->sk_lock.owned == 1)
1102 	 *
1103 	 * But following code is meant to be called from BH handlers,
1104 	 * so we should keep BH disabled, but early release socket ownership
1105 	 */
1106 	sock_release_ownership(sk);
1107 
1108 	if (flags & TCPF_WRITE_TIMER_DEFERRED) {
1109 		tcp_write_timer_handler(sk);
1110 		__sock_put(sk);
1111 	}
1112 	if (flags & TCPF_DELACK_TIMER_DEFERRED) {
1113 		tcp_delack_timer_handler(sk);
1114 		__sock_put(sk);
1115 	}
1116 	if (flags & TCPF_MTU_REDUCED_DEFERRED) {
1117 		inet_csk(sk)->icsk_af_ops->mtu_reduced(sk);
1118 		__sock_put(sk);
1119 	}
1120 }
1121 EXPORT_SYMBOL(tcp_release_cb);
1122 
1123 void __init tcp_tasklet_init(void)
1124 {
1125 	int i;
1126 
1127 	for_each_possible_cpu(i) {
1128 		struct tsq_tasklet *tsq = &per_cpu(tsq_tasklet, i);
1129 
1130 		INIT_LIST_HEAD(&tsq->head);
1131 		tasklet_setup(&tsq->tasklet, tcp_tasklet_func);
1132 	}
1133 }
1134 
1135 /*
1136  * Write buffer destructor automatically called from kfree_skb.
1137  * We can't xmit new skbs from this context, as we might already
1138  * hold qdisc lock.
1139  */
1140 void tcp_wfree(struct sk_buff *skb)
1141 {
1142 	struct sock *sk = skb->sk;
1143 	struct tcp_sock *tp = tcp_sk(sk);
1144 	unsigned long flags, nval, oval;
1145 
1146 	/* Keep one reference on sk_wmem_alloc.
1147 	 * Will be released by sk_free() from here or tcp_tasklet_func()
1148 	 */
1149 	WARN_ON(refcount_sub_and_test(skb->truesize - 1, &sk->sk_wmem_alloc));
1150 
1151 	/* If this softirq is serviced by ksoftirqd, we are likely under stress.
1152 	 * Wait until our queues (qdisc + devices) are drained.
1153 	 * This gives :
1154 	 * - less callbacks to tcp_write_xmit(), reducing stress (batches)
1155 	 * - chance for incoming ACK (processed by another cpu maybe)
1156 	 *   to migrate this flow (skb->ooo_okay will be eventually set)
1157 	 */
1158 	if (refcount_read(&sk->sk_wmem_alloc) >= SKB_TRUESIZE(1) && this_cpu_ksoftirqd() == current)
1159 		goto out;
1160 
1161 	for (oval = READ_ONCE(sk->sk_tsq_flags);; oval = nval) {
1162 		struct tsq_tasklet *tsq;
1163 		bool empty;
1164 
1165 		if (!(oval & TSQF_THROTTLED) || (oval & TSQF_QUEUED))
1166 			goto out;
1167 
1168 		nval = (oval & ~TSQF_THROTTLED) | TSQF_QUEUED;
1169 		nval = cmpxchg(&sk->sk_tsq_flags, oval, nval);
1170 		if (nval != oval)
1171 			continue;
1172 
1173 		/* queue this socket to tasklet queue */
1174 		local_irq_save(flags);
1175 		tsq = this_cpu_ptr(&tsq_tasklet);
1176 		empty = list_empty(&tsq->head);
1177 		list_add(&tp->tsq_node, &tsq->head);
1178 		if (empty)
1179 			tasklet_schedule(&tsq->tasklet);
1180 		local_irq_restore(flags);
1181 		return;
1182 	}
1183 out:
1184 	sk_free(sk);
1185 }
1186 
1187 /* Note: Called under soft irq.
1188  * We can call TCP stack right away, unless socket is owned by user.
1189  */
1190 enum hrtimer_restart tcp_pace_kick(struct hrtimer *timer)
1191 {
1192 	struct tcp_sock *tp = container_of(timer, struct tcp_sock, pacing_timer);
1193 	struct sock *sk = (struct sock *)tp;
1194 
1195 	tcp_tsq_handler(sk);
1196 	sock_put(sk);
1197 
1198 	return HRTIMER_NORESTART;
1199 }
1200 
1201 static void tcp_update_skb_after_send(struct sock *sk, struct sk_buff *skb,
1202 				      u64 prior_wstamp)
1203 {
1204 	struct tcp_sock *tp = tcp_sk(sk);
1205 
1206 	if (sk->sk_pacing_status != SK_PACING_NONE) {
1207 		unsigned long rate = sk->sk_pacing_rate;
1208 
1209 		/* Original sch_fq does not pace first 10 MSS
1210 		 * Note that tp->data_segs_out overflows after 2^32 packets,
1211 		 * this is a minor annoyance.
1212 		 */
1213 		if (rate != ~0UL && rate && tp->data_segs_out >= 10) {
1214 			u64 len_ns = div64_ul((u64)skb->len * NSEC_PER_SEC, rate);
1215 			u64 credit = tp->tcp_wstamp_ns - prior_wstamp;
1216 
1217 			/* take into account OS jitter */
1218 			len_ns -= min_t(u64, len_ns / 2, credit);
1219 			tp->tcp_wstamp_ns += len_ns;
1220 		}
1221 	}
1222 	list_move_tail(&skb->tcp_tsorted_anchor, &tp->tsorted_sent_queue);
1223 }
1224 
1225 INDIRECT_CALLABLE_DECLARE(int ip_queue_xmit(struct sock *sk, struct sk_buff *skb, struct flowi *fl));
1226 INDIRECT_CALLABLE_DECLARE(int inet6_csk_xmit(struct sock *sk, struct sk_buff *skb, struct flowi *fl));
1227 INDIRECT_CALLABLE_DECLARE(void tcp_v4_send_check(struct sock *sk, struct sk_buff *skb));
1228 
1229 /* This routine actually transmits TCP packets queued in by
1230  * tcp_do_sendmsg().  This is used by both the initial
1231  * transmission and possible later retransmissions.
1232  * All SKB's seen here are completely headerless.  It is our
1233  * job to build the TCP header, and pass the packet down to
1234  * IP so it can do the same plus pass the packet off to the
1235  * device.
1236  *
1237  * We are working here with either a clone of the original
1238  * SKB, or a fresh unique copy made by the retransmit engine.
1239  */
1240 static int __tcp_transmit_skb(struct sock *sk, struct sk_buff *skb,
1241 			      int clone_it, gfp_t gfp_mask, u32 rcv_nxt)
1242 {
1243 	const struct inet_connection_sock *icsk = inet_csk(sk);
1244 	struct inet_sock *inet;
1245 	struct tcp_sock *tp;
1246 	struct tcp_skb_cb *tcb;
1247 	struct tcp_out_options opts;
1248 	unsigned int tcp_options_size, tcp_header_size;
1249 	struct sk_buff *oskb = NULL;
1250 	struct tcp_md5sig_key *md5;
1251 	struct tcphdr *th;
1252 	u64 prior_wstamp;
1253 	int err;
1254 
1255 	BUG_ON(!skb || !tcp_skb_pcount(skb));
1256 	tp = tcp_sk(sk);
1257 	prior_wstamp = tp->tcp_wstamp_ns;
1258 	tp->tcp_wstamp_ns = max(tp->tcp_wstamp_ns, tp->tcp_clock_cache);
1259 	skb_set_delivery_time(skb, tp->tcp_wstamp_ns, true);
1260 	if (clone_it) {
1261 		oskb = skb;
1262 
1263 		tcp_skb_tsorted_save(oskb) {
1264 			if (unlikely(skb_cloned(oskb)))
1265 				skb = pskb_copy(oskb, gfp_mask);
1266 			else
1267 				skb = skb_clone(oskb, gfp_mask);
1268 		} tcp_skb_tsorted_restore(oskb);
1269 
1270 		if (unlikely(!skb))
1271 			return -ENOBUFS;
1272 		/* retransmit skbs might have a non zero value in skb->dev
1273 		 * because skb->dev is aliased with skb->rbnode.rb_left
1274 		 */
1275 		skb->dev = NULL;
1276 	}
1277 
1278 	inet = inet_sk(sk);
1279 	tcb = TCP_SKB_CB(skb);
1280 	memset(&opts, 0, sizeof(opts));
1281 
1282 	if (unlikely(tcb->tcp_flags & TCPHDR_SYN)) {
1283 		tcp_options_size = tcp_syn_options(sk, skb, &opts, &md5);
1284 	} else {
1285 		tcp_options_size = tcp_established_options(sk, skb, &opts,
1286 							   &md5);
1287 		/* Force a PSH flag on all (GSO) packets to expedite GRO flush
1288 		 * at receiver : This slightly improve GRO performance.
1289 		 * Note that we do not force the PSH flag for non GSO packets,
1290 		 * because they might be sent under high congestion events,
1291 		 * and in this case it is better to delay the delivery of 1-MSS
1292 		 * packets and thus the corresponding ACK packet that would
1293 		 * release the following packet.
1294 		 */
1295 		if (tcp_skb_pcount(skb) > 1)
1296 			tcb->tcp_flags |= TCPHDR_PSH;
1297 	}
1298 	tcp_header_size = tcp_options_size + sizeof(struct tcphdr);
1299 
1300 	/* if no packet is in qdisc/device queue, then allow XPS to select
1301 	 * another queue. We can be called from tcp_tsq_handler()
1302 	 * which holds one reference to sk.
1303 	 *
1304 	 * TODO: Ideally, in-flight pure ACK packets should not matter here.
1305 	 * One way to get this would be to set skb->truesize = 2 on them.
1306 	 */
1307 	skb->ooo_okay = sk_wmem_alloc_get(sk) < SKB_TRUESIZE(1);
1308 
1309 	/* If we had to use memory reserve to allocate this skb,
1310 	 * this might cause drops if packet is looped back :
1311 	 * Other socket might not have SOCK_MEMALLOC.
1312 	 * Packets not looped back do not care about pfmemalloc.
1313 	 */
1314 	skb->pfmemalloc = 0;
1315 
1316 	skb_push(skb, tcp_header_size);
1317 	skb_reset_transport_header(skb);
1318 
1319 	skb_orphan(skb);
1320 	skb->sk = sk;
1321 	skb->destructor = skb_is_tcp_pure_ack(skb) ? __sock_wfree : tcp_wfree;
1322 	refcount_add(skb->truesize, &sk->sk_wmem_alloc);
1323 
1324 	skb_set_dst_pending_confirm(skb, sk->sk_dst_pending_confirm);
1325 
1326 	/* Build TCP header and checksum it. */
1327 	th = (struct tcphdr *)skb->data;
1328 	th->source		= inet->inet_sport;
1329 	th->dest		= inet->inet_dport;
1330 	th->seq			= htonl(tcb->seq);
1331 	th->ack_seq		= htonl(rcv_nxt);
1332 	*(((__be16 *)th) + 6)	= htons(((tcp_header_size >> 2) << 12) |
1333 					tcb->tcp_flags);
1334 
1335 	th->check		= 0;
1336 	th->urg_ptr		= 0;
1337 
1338 	/* The urg_mode check is necessary during a below snd_una win probe */
1339 	if (unlikely(tcp_urg_mode(tp) && before(tcb->seq, tp->snd_up))) {
1340 		if (before(tp->snd_up, tcb->seq + 0x10000)) {
1341 			th->urg_ptr = htons(tp->snd_up - tcb->seq);
1342 			th->urg = 1;
1343 		} else if (after(tcb->seq + 0xFFFF, tp->snd_nxt)) {
1344 			th->urg_ptr = htons(0xFFFF);
1345 			th->urg = 1;
1346 		}
1347 	}
1348 
1349 	skb_shinfo(skb)->gso_type = sk->sk_gso_type;
1350 	if (likely(!(tcb->tcp_flags & TCPHDR_SYN))) {
1351 		th->window      = htons(tcp_select_window(sk));
1352 		tcp_ecn_send(sk, skb, th, tcp_header_size);
1353 	} else {
1354 		/* RFC1323: The window in SYN & SYN/ACK segments
1355 		 * is never scaled.
1356 		 */
1357 		th->window	= htons(min(tp->rcv_wnd, 65535U));
1358 	}
1359 
1360 	tcp_options_write(th, tp, &opts);
1361 
1362 #ifdef CONFIG_TCP_MD5SIG
1363 	/* Calculate the MD5 hash, as we have all we need now */
1364 	if (md5) {
1365 		sk_gso_disable(sk);
1366 		tp->af_specific->calc_md5_hash(opts.hash_location,
1367 					       md5, sk, skb);
1368 	}
1369 #endif
1370 
1371 	/* BPF prog is the last one writing header option */
1372 	bpf_skops_write_hdr_opt(sk, skb, NULL, NULL, 0, &opts);
1373 
1374 	INDIRECT_CALL_INET(icsk->icsk_af_ops->send_check,
1375 			   tcp_v6_send_check, tcp_v4_send_check,
1376 			   sk, skb);
1377 
1378 	if (likely(tcb->tcp_flags & TCPHDR_ACK))
1379 		tcp_event_ack_sent(sk, tcp_skb_pcount(skb), rcv_nxt);
1380 
1381 	if (skb->len != tcp_header_size) {
1382 		tcp_event_data_sent(tp, sk);
1383 		tp->data_segs_out += tcp_skb_pcount(skb);
1384 		tp->bytes_sent += skb->len - tcp_header_size;
1385 	}
1386 
1387 	if (after(tcb->end_seq, tp->snd_nxt) || tcb->seq == tcb->end_seq)
1388 		TCP_ADD_STATS(sock_net(sk), TCP_MIB_OUTSEGS,
1389 			      tcp_skb_pcount(skb));
1390 
1391 	tp->segs_out += tcp_skb_pcount(skb);
1392 	skb_set_hash_from_sk(skb, sk);
1393 	/* OK, its time to fill skb_shinfo(skb)->gso_{segs|size} */
1394 	skb_shinfo(skb)->gso_segs = tcp_skb_pcount(skb);
1395 	skb_shinfo(skb)->gso_size = tcp_skb_mss(skb);
1396 
1397 	/* Leave earliest departure time in skb->tstamp (skb->skb_mstamp_ns) */
1398 
1399 	/* Cleanup our debris for IP stacks */
1400 	memset(skb->cb, 0, max(sizeof(struct inet_skb_parm),
1401 			       sizeof(struct inet6_skb_parm)));
1402 
1403 	tcp_add_tx_delay(skb, tp);
1404 
1405 	err = INDIRECT_CALL_INET(icsk->icsk_af_ops->queue_xmit,
1406 				 inet6_csk_xmit, ip_queue_xmit,
1407 				 sk, skb, &inet->cork.fl);
1408 
1409 	if (unlikely(err > 0)) {
1410 		tcp_enter_cwr(sk);
1411 		err = net_xmit_eval(err);
1412 	}
1413 	if (!err && oskb) {
1414 		tcp_update_skb_after_send(sk, oskb, prior_wstamp);
1415 		tcp_rate_skb_sent(sk, oskb);
1416 	}
1417 	return err;
1418 }
1419 
1420 static int tcp_transmit_skb(struct sock *sk, struct sk_buff *skb, int clone_it,
1421 			    gfp_t gfp_mask)
1422 {
1423 	return __tcp_transmit_skb(sk, skb, clone_it, gfp_mask,
1424 				  tcp_sk(sk)->rcv_nxt);
1425 }
1426 
1427 /* This routine just queues the buffer for sending.
1428  *
1429  * NOTE: probe0 timer is not checked, do not forget tcp_push_pending_frames,
1430  * otherwise socket can stall.
1431  */
1432 static void tcp_queue_skb(struct sock *sk, struct sk_buff *skb)
1433 {
1434 	struct tcp_sock *tp = tcp_sk(sk);
1435 
1436 	/* Advance write_seq and place onto the write_queue. */
1437 	WRITE_ONCE(tp->write_seq, TCP_SKB_CB(skb)->end_seq);
1438 	__skb_header_release(skb);
1439 	tcp_add_write_queue_tail(sk, skb);
1440 	sk_wmem_queued_add(sk, skb->truesize);
1441 	sk_mem_charge(sk, skb->truesize);
1442 }
1443 
1444 /* Initialize TSO segments for a packet. */
1445 static void tcp_set_skb_tso_segs(struct sk_buff *skb, unsigned int mss_now)
1446 {
1447 	if (skb->len <= mss_now) {
1448 		/* Avoid the costly divide in the normal
1449 		 * non-TSO case.
1450 		 */
1451 		tcp_skb_pcount_set(skb, 1);
1452 		TCP_SKB_CB(skb)->tcp_gso_size = 0;
1453 	} else {
1454 		tcp_skb_pcount_set(skb, DIV_ROUND_UP(skb->len, mss_now));
1455 		TCP_SKB_CB(skb)->tcp_gso_size = mss_now;
1456 	}
1457 }
1458 
1459 /* Pcount in the middle of the write queue got changed, we need to do various
1460  * tweaks to fix counters
1461  */
1462 static void tcp_adjust_pcount(struct sock *sk, const struct sk_buff *skb, int decr)
1463 {
1464 	struct tcp_sock *tp = tcp_sk(sk);
1465 
1466 	tp->packets_out -= decr;
1467 
1468 	if (TCP_SKB_CB(skb)->sacked & TCPCB_SACKED_ACKED)
1469 		tp->sacked_out -= decr;
1470 	if (TCP_SKB_CB(skb)->sacked & TCPCB_SACKED_RETRANS)
1471 		tp->retrans_out -= decr;
1472 	if (TCP_SKB_CB(skb)->sacked & TCPCB_LOST)
1473 		tp->lost_out -= decr;
1474 
1475 	/* Reno case is special. Sigh... */
1476 	if (tcp_is_reno(tp) && decr > 0)
1477 		tp->sacked_out -= min_t(u32, tp->sacked_out, decr);
1478 
1479 	if (tp->lost_skb_hint &&
1480 	    before(TCP_SKB_CB(skb)->seq, TCP_SKB_CB(tp->lost_skb_hint)->seq) &&
1481 	    (TCP_SKB_CB(skb)->sacked & TCPCB_SACKED_ACKED))
1482 		tp->lost_cnt_hint -= decr;
1483 
1484 	tcp_verify_left_out(tp);
1485 }
1486 
1487 static bool tcp_has_tx_tstamp(const struct sk_buff *skb)
1488 {
1489 	return TCP_SKB_CB(skb)->txstamp_ack ||
1490 		(skb_shinfo(skb)->tx_flags & SKBTX_ANY_TSTAMP);
1491 }
1492 
1493 static void tcp_fragment_tstamp(struct sk_buff *skb, struct sk_buff *skb2)
1494 {
1495 	struct skb_shared_info *shinfo = skb_shinfo(skb);
1496 
1497 	if (unlikely(tcp_has_tx_tstamp(skb)) &&
1498 	    !before(shinfo->tskey, TCP_SKB_CB(skb2)->seq)) {
1499 		struct skb_shared_info *shinfo2 = skb_shinfo(skb2);
1500 		u8 tsflags = shinfo->tx_flags & SKBTX_ANY_TSTAMP;
1501 
1502 		shinfo->tx_flags &= ~tsflags;
1503 		shinfo2->tx_flags |= tsflags;
1504 		swap(shinfo->tskey, shinfo2->tskey);
1505 		TCP_SKB_CB(skb2)->txstamp_ack = TCP_SKB_CB(skb)->txstamp_ack;
1506 		TCP_SKB_CB(skb)->txstamp_ack = 0;
1507 	}
1508 }
1509 
1510 static void tcp_skb_fragment_eor(struct sk_buff *skb, struct sk_buff *skb2)
1511 {
1512 	TCP_SKB_CB(skb2)->eor = TCP_SKB_CB(skb)->eor;
1513 	TCP_SKB_CB(skb)->eor = 0;
1514 }
1515 
1516 /* Insert buff after skb on the write or rtx queue of sk.  */
1517 static void tcp_insert_write_queue_after(struct sk_buff *skb,
1518 					 struct sk_buff *buff,
1519 					 struct sock *sk,
1520 					 enum tcp_queue tcp_queue)
1521 {
1522 	if (tcp_queue == TCP_FRAG_IN_WRITE_QUEUE)
1523 		__skb_queue_after(&sk->sk_write_queue, skb, buff);
1524 	else
1525 		tcp_rbtree_insert(&sk->tcp_rtx_queue, buff);
1526 }
1527 
1528 /* Function to create two new TCP segments.  Shrinks the given segment
1529  * to the specified size and appends a new segment with the rest of the
1530  * packet to the list.  This won't be called frequently, I hope.
1531  * Remember, these are still headerless SKBs at this point.
1532  */
1533 int tcp_fragment(struct sock *sk, enum tcp_queue tcp_queue,
1534 		 struct sk_buff *skb, u32 len,
1535 		 unsigned int mss_now, gfp_t gfp)
1536 {
1537 	struct tcp_sock *tp = tcp_sk(sk);
1538 	struct sk_buff *buff;
1539 	int nsize, old_factor;
1540 	long limit;
1541 	int nlen;
1542 	u8 flags;
1543 
1544 	if (WARN_ON(len > skb->len))
1545 		return -EINVAL;
1546 
1547 	nsize = skb_headlen(skb) - len;
1548 	if (nsize < 0)
1549 		nsize = 0;
1550 
1551 	/* tcp_sendmsg() can overshoot sk_wmem_queued by one full size skb.
1552 	 * We need some allowance to not penalize applications setting small
1553 	 * SO_SNDBUF values.
1554 	 * Also allow first and last skb in retransmit queue to be split.
1555 	 */
1556 	limit = sk->sk_sndbuf + 2 * SKB_TRUESIZE(GSO_LEGACY_MAX_SIZE);
1557 	if (unlikely((sk->sk_wmem_queued >> 1) > limit &&
1558 		     tcp_queue != TCP_FRAG_IN_WRITE_QUEUE &&
1559 		     skb != tcp_rtx_queue_head(sk) &&
1560 		     skb != tcp_rtx_queue_tail(sk))) {
1561 		NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPWQUEUETOOBIG);
1562 		return -ENOMEM;
1563 	}
1564 
1565 	if (skb_unclone_keeptruesize(skb, gfp))
1566 		return -ENOMEM;
1567 
1568 	/* Get a new skb... force flag on. */
1569 	buff = tcp_stream_alloc_skb(sk, nsize, gfp, true);
1570 	if (!buff)
1571 		return -ENOMEM; /* We'll just try again later. */
1572 	skb_copy_decrypted(buff, skb);
1573 	mptcp_skb_ext_copy(buff, skb);
1574 
1575 	sk_wmem_queued_add(sk, buff->truesize);
1576 	sk_mem_charge(sk, buff->truesize);
1577 	nlen = skb->len - len - nsize;
1578 	buff->truesize += nlen;
1579 	skb->truesize -= nlen;
1580 
1581 	/* Correct the sequence numbers. */
1582 	TCP_SKB_CB(buff)->seq = TCP_SKB_CB(skb)->seq + len;
1583 	TCP_SKB_CB(buff)->end_seq = TCP_SKB_CB(skb)->end_seq;
1584 	TCP_SKB_CB(skb)->end_seq = TCP_SKB_CB(buff)->seq;
1585 
1586 	/* PSH and FIN should only be set in the second packet. */
1587 	flags = TCP_SKB_CB(skb)->tcp_flags;
1588 	TCP_SKB_CB(skb)->tcp_flags = flags & ~(TCPHDR_FIN | TCPHDR_PSH);
1589 	TCP_SKB_CB(buff)->tcp_flags = flags;
1590 	TCP_SKB_CB(buff)->sacked = TCP_SKB_CB(skb)->sacked;
1591 	tcp_skb_fragment_eor(skb, buff);
1592 
1593 	skb_split(skb, buff, len);
1594 
1595 	skb_set_delivery_time(buff, skb->tstamp, true);
1596 	tcp_fragment_tstamp(skb, buff);
1597 
1598 	old_factor = tcp_skb_pcount(skb);
1599 
1600 	/* Fix up tso_factor for both original and new SKB.  */
1601 	tcp_set_skb_tso_segs(skb, mss_now);
1602 	tcp_set_skb_tso_segs(buff, mss_now);
1603 
1604 	/* Update delivered info for the new segment */
1605 	TCP_SKB_CB(buff)->tx = TCP_SKB_CB(skb)->tx;
1606 
1607 	/* If this packet has been sent out already, we must
1608 	 * adjust the various packet counters.
1609 	 */
1610 	if (!before(tp->snd_nxt, TCP_SKB_CB(buff)->end_seq)) {
1611 		int diff = old_factor - tcp_skb_pcount(skb) -
1612 			tcp_skb_pcount(buff);
1613 
1614 		if (diff)
1615 			tcp_adjust_pcount(sk, skb, diff);
1616 	}
1617 
1618 	/* Link BUFF into the send queue. */
1619 	__skb_header_release(buff);
1620 	tcp_insert_write_queue_after(skb, buff, sk, tcp_queue);
1621 	if (tcp_queue == TCP_FRAG_IN_RTX_QUEUE)
1622 		list_add(&buff->tcp_tsorted_anchor, &skb->tcp_tsorted_anchor);
1623 
1624 	return 0;
1625 }
1626 
1627 /* This is similar to __pskb_pull_tail(). The difference is that pulled
1628  * data is not copied, but immediately discarded.
1629  */
1630 static int __pskb_trim_head(struct sk_buff *skb, int len)
1631 {
1632 	struct skb_shared_info *shinfo;
1633 	int i, k, eat;
1634 
1635 	eat = min_t(int, len, skb_headlen(skb));
1636 	if (eat) {
1637 		__skb_pull(skb, eat);
1638 		len -= eat;
1639 		if (!len)
1640 			return 0;
1641 	}
1642 	eat = len;
1643 	k = 0;
1644 	shinfo = skb_shinfo(skb);
1645 	for (i = 0; i < shinfo->nr_frags; i++) {
1646 		int size = skb_frag_size(&shinfo->frags[i]);
1647 
1648 		if (size <= eat) {
1649 			skb_frag_unref(skb, i);
1650 			eat -= size;
1651 		} else {
1652 			shinfo->frags[k] = shinfo->frags[i];
1653 			if (eat) {
1654 				skb_frag_off_add(&shinfo->frags[k], eat);
1655 				skb_frag_size_sub(&shinfo->frags[k], eat);
1656 				eat = 0;
1657 			}
1658 			k++;
1659 		}
1660 	}
1661 	shinfo->nr_frags = k;
1662 
1663 	skb->data_len -= len;
1664 	skb->len = skb->data_len;
1665 	return len;
1666 }
1667 
1668 /* Remove acked data from a packet in the transmit queue. */
1669 int tcp_trim_head(struct sock *sk, struct sk_buff *skb, u32 len)
1670 {
1671 	u32 delta_truesize;
1672 
1673 	if (skb_unclone_keeptruesize(skb, GFP_ATOMIC))
1674 		return -ENOMEM;
1675 
1676 	delta_truesize = __pskb_trim_head(skb, len);
1677 
1678 	TCP_SKB_CB(skb)->seq += len;
1679 
1680 	if (delta_truesize) {
1681 		skb->truesize	   -= delta_truesize;
1682 		sk_wmem_queued_add(sk, -delta_truesize);
1683 		if (!skb_zcopy_pure(skb))
1684 			sk_mem_uncharge(sk, delta_truesize);
1685 	}
1686 
1687 	/* Any change of skb->len requires recalculation of tso factor. */
1688 	if (tcp_skb_pcount(skb) > 1)
1689 		tcp_set_skb_tso_segs(skb, tcp_skb_mss(skb));
1690 
1691 	return 0;
1692 }
1693 
1694 /* Calculate MSS not accounting any TCP options.  */
1695 static inline int __tcp_mtu_to_mss(struct sock *sk, int pmtu)
1696 {
1697 	const struct tcp_sock *tp = tcp_sk(sk);
1698 	const struct inet_connection_sock *icsk = inet_csk(sk);
1699 	int mss_now;
1700 
1701 	/* Calculate base mss without TCP options:
1702 	   It is MMS_S - sizeof(tcphdr) of rfc1122
1703 	 */
1704 	mss_now = pmtu - icsk->icsk_af_ops->net_header_len - sizeof(struct tcphdr);
1705 
1706 	/* IPv6 adds a frag_hdr in case RTAX_FEATURE_ALLFRAG is set */
1707 	if (icsk->icsk_af_ops->net_frag_header_len) {
1708 		const struct dst_entry *dst = __sk_dst_get(sk);
1709 
1710 		if (dst && dst_allfrag(dst))
1711 			mss_now -= icsk->icsk_af_ops->net_frag_header_len;
1712 	}
1713 
1714 	/* Clamp it (mss_clamp does not include tcp options) */
1715 	if (mss_now > tp->rx_opt.mss_clamp)
1716 		mss_now = tp->rx_opt.mss_clamp;
1717 
1718 	/* Now subtract optional transport overhead */
1719 	mss_now -= icsk->icsk_ext_hdr_len;
1720 
1721 	/* Then reserve room for full set of TCP options and 8 bytes of data */
1722 	mss_now = max(mss_now,
1723 		      READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_min_snd_mss));
1724 	return mss_now;
1725 }
1726 
1727 /* Calculate MSS. Not accounting for SACKs here.  */
1728 int tcp_mtu_to_mss(struct sock *sk, int pmtu)
1729 {
1730 	/* Subtract TCP options size, not including SACKs */
1731 	return __tcp_mtu_to_mss(sk, pmtu) -
1732 	       (tcp_sk(sk)->tcp_header_len - sizeof(struct tcphdr));
1733 }
1734 EXPORT_SYMBOL(tcp_mtu_to_mss);
1735 
1736 /* Inverse of above */
1737 int tcp_mss_to_mtu(struct sock *sk, int mss)
1738 {
1739 	const struct tcp_sock *tp = tcp_sk(sk);
1740 	const struct inet_connection_sock *icsk = inet_csk(sk);
1741 	int mtu;
1742 
1743 	mtu = mss +
1744 	      tp->tcp_header_len +
1745 	      icsk->icsk_ext_hdr_len +
1746 	      icsk->icsk_af_ops->net_header_len;
1747 
1748 	/* IPv6 adds a frag_hdr in case RTAX_FEATURE_ALLFRAG is set */
1749 	if (icsk->icsk_af_ops->net_frag_header_len) {
1750 		const struct dst_entry *dst = __sk_dst_get(sk);
1751 
1752 		if (dst && dst_allfrag(dst))
1753 			mtu += icsk->icsk_af_ops->net_frag_header_len;
1754 	}
1755 	return mtu;
1756 }
1757 EXPORT_SYMBOL(tcp_mss_to_mtu);
1758 
1759 /* MTU probing init per socket */
1760 void tcp_mtup_init(struct sock *sk)
1761 {
1762 	struct tcp_sock *tp = tcp_sk(sk);
1763 	struct inet_connection_sock *icsk = inet_csk(sk);
1764 	struct net *net = sock_net(sk);
1765 
1766 	icsk->icsk_mtup.enabled = READ_ONCE(net->ipv4.sysctl_tcp_mtu_probing) > 1;
1767 	icsk->icsk_mtup.search_high = tp->rx_opt.mss_clamp + sizeof(struct tcphdr) +
1768 			       icsk->icsk_af_ops->net_header_len;
1769 	icsk->icsk_mtup.search_low = tcp_mss_to_mtu(sk, READ_ONCE(net->ipv4.sysctl_tcp_base_mss));
1770 	icsk->icsk_mtup.probe_size = 0;
1771 	if (icsk->icsk_mtup.enabled)
1772 		icsk->icsk_mtup.probe_timestamp = tcp_jiffies32;
1773 }
1774 EXPORT_SYMBOL(tcp_mtup_init);
1775 
1776 /* This function synchronize snd mss to current pmtu/exthdr set.
1777 
1778    tp->rx_opt.user_mss is mss set by user by TCP_MAXSEG. It does NOT counts
1779    for TCP options, but includes only bare TCP header.
1780 
1781    tp->rx_opt.mss_clamp is mss negotiated at connection setup.
1782    It is minimum of user_mss and mss received with SYN.
1783    It also does not include TCP options.
1784 
1785    inet_csk(sk)->icsk_pmtu_cookie is last pmtu, seen by this function.
1786 
1787    tp->mss_cache is current effective sending mss, including
1788    all tcp options except for SACKs. It is evaluated,
1789    taking into account current pmtu, but never exceeds
1790    tp->rx_opt.mss_clamp.
1791 
1792    NOTE1. rfc1122 clearly states that advertised MSS
1793    DOES NOT include either tcp or ip options.
1794 
1795    NOTE2. inet_csk(sk)->icsk_pmtu_cookie and tp->mss_cache
1796    are READ ONLY outside this function.		--ANK (980731)
1797  */
1798 unsigned int tcp_sync_mss(struct sock *sk, u32 pmtu)
1799 {
1800 	struct tcp_sock *tp = tcp_sk(sk);
1801 	struct inet_connection_sock *icsk = inet_csk(sk);
1802 	int mss_now;
1803 
1804 	if (icsk->icsk_mtup.search_high > pmtu)
1805 		icsk->icsk_mtup.search_high = pmtu;
1806 
1807 	mss_now = tcp_mtu_to_mss(sk, pmtu);
1808 	mss_now = tcp_bound_to_half_wnd(tp, mss_now);
1809 
1810 	/* And store cached results */
1811 	icsk->icsk_pmtu_cookie = pmtu;
1812 	if (icsk->icsk_mtup.enabled)
1813 		mss_now = min(mss_now, tcp_mtu_to_mss(sk, icsk->icsk_mtup.search_low));
1814 	tp->mss_cache = mss_now;
1815 
1816 	return mss_now;
1817 }
1818 EXPORT_SYMBOL(tcp_sync_mss);
1819 
1820 /* Compute the current effective MSS, taking SACKs and IP options,
1821  * and even PMTU discovery events into account.
1822  */
1823 unsigned int tcp_current_mss(struct sock *sk)
1824 {
1825 	const struct tcp_sock *tp = tcp_sk(sk);
1826 	const struct dst_entry *dst = __sk_dst_get(sk);
1827 	u32 mss_now;
1828 	unsigned int header_len;
1829 	struct tcp_out_options opts;
1830 	struct tcp_md5sig_key *md5;
1831 
1832 	mss_now = tp->mss_cache;
1833 
1834 	if (dst) {
1835 		u32 mtu = dst_mtu(dst);
1836 		if (mtu != inet_csk(sk)->icsk_pmtu_cookie)
1837 			mss_now = tcp_sync_mss(sk, mtu);
1838 	}
1839 
1840 	header_len = tcp_established_options(sk, NULL, &opts, &md5) +
1841 		     sizeof(struct tcphdr);
1842 	/* The mss_cache is sized based on tp->tcp_header_len, which assumes
1843 	 * some common options. If this is an odd packet (because we have SACK
1844 	 * blocks etc) then our calculated header_len will be different, and
1845 	 * we have to adjust mss_now correspondingly */
1846 	if (header_len != tp->tcp_header_len) {
1847 		int delta = (int) header_len - tp->tcp_header_len;
1848 		mss_now -= delta;
1849 	}
1850 
1851 	return mss_now;
1852 }
1853 
1854 /* RFC2861, slow part. Adjust cwnd, after it was not full during one rto.
1855  * As additional protections, we do not touch cwnd in retransmission phases,
1856  * and if application hit its sndbuf limit recently.
1857  */
1858 static void tcp_cwnd_application_limited(struct sock *sk)
1859 {
1860 	struct tcp_sock *tp = tcp_sk(sk);
1861 
1862 	if (inet_csk(sk)->icsk_ca_state == TCP_CA_Open &&
1863 	    sk->sk_socket && !test_bit(SOCK_NOSPACE, &sk->sk_socket->flags)) {
1864 		/* Limited by application or receiver window. */
1865 		u32 init_win = tcp_init_cwnd(tp, __sk_dst_get(sk));
1866 		u32 win_used = max(tp->snd_cwnd_used, init_win);
1867 		if (win_used < tcp_snd_cwnd(tp)) {
1868 			tp->snd_ssthresh = tcp_current_ssthresh(sk);
1869 			tcp_snd_cwnd_set(tp, (tcp_snd_cwnd(tp) + win_used) >> 1);
1870 		}
1871 		tp->snd_cwnd_used = 0;
1872 	}
1873 	tp->snd_cwnd_stamp = tcp_jiffies32;
1874 }
1875 
1876 static void tcp_cwnd_validate(struct sock *sk, bool is_cwnd_limited)
1877 {
1878 	const struct tcp_congestion_ops *ca_ops = inet_csk(sk)->icsk_ca_ops;
1879 	struct tcp_sock *tp = tcp_sk(sk);
1880 
1881 	/* Track the maximum number of outstanding packets in each
1882 	 * window, and remember whether we were cwnd-limited then.
1883 	 */
1884 	if (!before(tp->snd_una, tp->max_packets_seq) ||
1885 	    tp->packets_out > tp->max_packets_out ||
1886 	    is_cwnd_limited) {
1887 		tp->max_packets_out = tp->packets_out;
1888 		tp->max_packets_seq = tp->snd_nxt;
1889 		tp->is_cwnd_limited = is_cwnd_limited;
1890 	}
1891 
1892 	if (tcp_is_cwnd_limited(sk)) {
1893 		/* Network is feed fully. */
1894 		tp->snd_cwnd_used = 0;
1895 		tp->snd_cwnd_stamp = tcp_jiffies32;
1896 	} else {
1897 		/* Network starves. */
1898 		if (tp->packets_out > tp->snd_cwnd_used)
1899 			tp->snd_cwnd_used = tp->packets_out;
1900 
1901 		if (READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_slow_start_after_idle) &&
1902 		    (s32)(tcp_jiffies32 - tp->snd_cwnd_stamp) >= inet_csk(sk)->icsk_rto &&
1903 		    !ca_ops->cong_control)
1904 			tcp_cwnd_application_limited(sk);
1905 
1906 		/* The following conditions together indicate the starvation
1907 		 * is caused by insufficient sender buffer:
1908 		 * 1) just sent some data (see tcp_write_xmit)
1909 		 * 2) not cwnd limited (this else condition)
1910 		 * 3) no more data to send (tcp_write_queue_empty())
1911 		 * 4) application is hitting buffer limit (SOCK_NOSPACE)
1912 		 */
1913 		if (tcp_write_queue_empty(sk) && sk->sk_socket &&
1914 		    test_bit(SOCK_NOSPACE, &sk->sk_socket->flags) &&
1915 		    (1 << sk->sk_state) & (TCPF_ESTABLISHED | TCPF_CLOSE_WAIT))
1916 			tcp_chrono_start(sk, TCP_CHRONO_SNDBUF_LIMITED);
1917 	}
1918 }
1919 
1920 /* Minshall's variant of the Nagle send check. */
1921 static bool tcp_minshall_check(const struct tcp_sock *tp)
1922 {
1923 	return after(tp->snd_sml, tp->snd_una) &&
1924 		!after(tp->snd_sml, tp->snd_nxt);
1925 }
1926 
1927 /* Update snd_sml if this skb is under mss
1928  * Note that a TSO packet might end with a sub-mss segment
1929  * The test is really :
1930  * if ((skb->len % mss) != 0)
1931  *        tp->snd_sml = TCP_SKB_CB(skb)->end_seq;
1932  * But we can avoid doing the divide again given we already have
1933  *  skb_pcount = skb->len / mss_now
1934  */
1935 static void tcp_minshall_update(struct tcp_sock *tp, unsigned int mss_now,
1936 				const struct sk_buff *skb)
1937 {
1938 	if (skb->len < tcp_skb_pcount(skb) * mss_now)
1939 		tp->snd_sml = TCP_SKB_CB(skb)->end_seq;
1940 }
1941 
1942 /* Return false, if packet can be sent now without violation Nagle's rules:
1943  * 1. It is full sized. (provided by caller in %partial bool)
1944  * 2. Or it contains FIN. (already checked by caller)
1945  * 3. Or TCP_CORK is not set, and TCP_NODELAY is set.
1946  * 4. Or TCP_CORK is not set, and all sent packets are ACKed.
1947  *    With Minshall's modification: all sent small packets are ACKed.
1948  */
1949 static bool tcp_nagle_check(bool partial, const struct tcp_sock *tp,
1950 			    int nonagle)
1951 {
1952 	return partial &&
1953 		((nonagle & TCP_NAGLE_CORK) ||
1954 		 (!nonagle && tp->packets_out && tcp_minshall_check(tp)));
1955 }
1956 
1957 /* Return how many segs we'd like on a TSO packet,
1958  * depending on current pacing rate, and how close the peer is.
1959  *
1960  * Rationale is:
1961  * - For close peers, we rather send bigger packets to reduce
1962  *   cpu costs, because occasional losses will be repaired fast.
1963  * - For long distance/rtt flows, we would like to get ACK clocking
1964  *   with 1 ACK per ms.
1965  *
1966  * Use min_rtt to help adapt TSO burst size, with smaller min_rtt resulting
1967  * in bigger TSO bursts. We we cut the RTT-based allowance in half
1968  * for every 2^9 usec (aka 512 us) of RTT, so that the RTT-based allowance
1969  * is below 1500 bytes after 6 * ~500 usec = 3ms.
1970  */
1971 static u32 tcp_tso_autosize(const struct sock *sk, unsigned int mss_now,
1972 			    int min_tso_segs)
1973 {
1974 	unsigned long bytes;
1975 	u32 r;
1976 
1977 	bytes = sk->sk_pacing_rate >> READ_ONCE(sk->sk_pacing_shift);
1978 
1979 	r = tcp_min_rtt(tcp_sk(sk)) >> sock_net(sk)->ipv4.sysctl_tcp_tso_rtt_log;
1980 	if (r < BITS_PER_TYPE(sk->sk_gso_max_size))
1981 		bytes += sk->sk_gso_max_size >> r;
1982 
1983 	bytes = min_t(unsigned long, bytes, sk->sk_gso_max_size);
1984 
1985 	return max_t(u32, bytes / mss_now, min_tso_segs);
1986 }
1987 
1988 /* Return the number of segments we want in the skb we are transmitting.
1989  * See if congestion control module wants to decide; otherwise, autosize.
1990  */
1991 static u32 tcp_tso_segs(struct sock *sk, unsigned int mss_now)
1992 {
1993 	const struct tcp_congestion_ops *ca_ops = inet_csk(sk)->icsk_ca_ops;
1994 	u32 min_tso, tso_segs;
1995 
1996 	min_tso = ca_ops->min_tso_segs ?
1997 			ca_ops->min_tso_segs(sk) :
1998 			sock_net(sk)->ipv4.sysctl_tcp_min_tso_segs;
1999 
2000 	tso_segs = tcp_tso_autosize(sk, mss_now, min_tso);
2001 	return min_t(u32, tso_segs, sk->sk_gso_max_segs);
2002 }
2003 
2004 /* Returns the portion of skb which can be sent right away */
2005 static unsigned int tcp_mss_split_point(const struct sock *sk,
2006 					const struct sk_buff *skb,
2007 					unsigned int mss_now,
2008 					unsigned int max_segs,
2009 					int nonagle)
2010 {
2011 	const struct tcp_sock *tp = tcp_sk(sk);
2012 	u32 partial, needed, window, max_len;
2013 
2014 	window = tcp_wnd_end(tp) - TCP_SKB_CB(skb)->seq;
2015 	max_len = mss_now * max_segs;
2016 
2017 	if (likely(max_len <= window && skb != tcp_write_queue_tail(sk)))
2018 		return max_len;
2019 
2020 	needed = min(skb->len, window);
2021 
2022 	if (max_len <= needed)
2023 		return max_len;
2024 
2025 	partial = needed % mss_now;
2026 	/* If last segment is not a full MSS, check if Nagle rules allow us
2027 	 * to include this last segment in this skb.
2028 	 * Otherwise, we'll split the skb at last MSS boundary
2029 	 */
2030 	if (tcp_nagle_check(partial != 0, tp, nonagle))
2031 		return needed - partial;
2032 
2033 	return needed;
2034 }
2035 
2036 /* Can at least one segment of SKB be sent right now, according to the
2037  * congestion window rules?  If so, return how many segments are allowed.
2038  */
2039 static inline unsigned int tcp_cwnd_test(const struct tcp_sock *tp,
2040 					 const struct sk_buff *skb)
2041 {
2042 	u32 in_flight, cwnd, halfcwnd;
2043 
2044 	/* Don't be strict about the congestion window for the final FIN.  */
2045 	if ((TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN) &&
2046 	    tcp_skb_pcount(skb) == 1)
2047 		return 1;
2048 
2049 	in_flight = tcp_packets_in_flight(tp);
2050 	cwnd = tcp_snd_cwnd(tp);
2051 	if (in_flight >= cwnd)
2052 		return 0;
2053 
2054 	/* For better scheduling, ensure we have at least
2055 	 * 2 GSO packets in flight.
2056 	 */
2057 	halfcwnd = max(cwnd >> 1, 1U);
2058 	return min(halfcwnd, cwnd - in_flight);
2059 }
2060 
2061 /* Initialize TSO state of a skb.
2062  * This must be invoked the first time we consider transmitting
2063  * SKB onto the wire.
2064  */
2065 static int tcp_init_tso_segs(struct sk_buff *skb, unsigned int mss_now)
2066 {
2067 	int tso_segs = tcp_skb_pcount(skb);
2068 
2069 	if (!tso_segs || (tso_segs > 1 && tcp_skb_mss(skb) != mss_now)) {
2070 		tcp_set_skb_tso_segs(skb, mss_now);
2071 		tso_segs = tcp_skb_pcount(skb);
2072 	}
2073 	return tso_segs;
2074 }
2075 
2076 
2077 /* Return true if the Nagle test allows this packet to be
2078  * sent now.
2079  */
2080 static inline bool tcp_nagle_test(const struct tcp_sock *tp, const struct sk_buff *skb,
2081 				  unsigned int cur_mss, int nonagle)
2082 {
2083 	/* Nagle rule does not apply to frames, which sit in the middle of the
2084 	 * write_queue (they have no chances to get new data).
2085 	 *
2086 	 * This is implemented in the callers, where they modify the 'nonagle'
2087 	 * argument based upon the location of SKB in the send queue.
2088 	 */
2089 	if (nonagle & TCP_NAGLE_PUSH)
2090 		return true;
2091 
2092 	/* Don't use the nagle rule for urgent data (or for the final FIN). */
2093 	if (tcp_urg_mode(tp) || (TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN))
2094 		return true;
2095 
2096 	if (!tcp_nagle_check(skb->len < cur_mss, tp, nonagle))
2097 		return true;
2098 
2099 	return false;
2100 }
2101 
2102 /* Does at least the first segment of SKB fit into the send window? */
2103 static bool tcp_snd_wnd_test(const struct tcp_sock *tp,
2104 			     const struct sk_buff *skb,
2105 			     unsigned int cur_mss)
2106 {
2107 	u32 end_seq = TCP_SKB_CB(skb)->end_seq;
2108 
2109 	if (skb->len > cur_mss)
2110 		end_seq = TCP_SKB_CB(skb)->seq + cur_mss;
2111 
2112 	return !after(end_seq, tcp_wnd_end(tp));
2113 }
2114 
2115 /* Trim TSO SKB to LEN bytes, put the remaining data into a new packet
2116  * which is put after SKB on the list.  It is very much like
2117  * tcp_fragment() except that it may make several kinds of assumptions
2118  * in order to speed up the splitting operation.  In particular, we
2119  * know that all the data is in scatter-gather pages, and that the
2120  * packet has never been sent out before (and thus is not cloned).
2121  */
2122 static int tso_fragment(struct sock *sk, struct sk_buff *skb, unsigned int len,
2123 			unsigned int mss_now, gfp_t gfp)
2124 {
2125 	int nlen = skb->len - len;
2126 	struct sk_buff *buff;
2127 	u8 flags;
2128 
2129 	/* All of a TSO frame must be composed of paged data.  */
2130 	if (skb->len != skb->data_len)
2131 		return tcp_fragment(sk, TCP_FRAG_IN_WRITE_QUEUE,
2132 				    skb, len, mss_now, gfp);
2133 
2134 	buff = tcp_stream_alloc_skb(sk, 0, gfp, true);
2135 	if (unlikely(!buff))
2136 		return -ENOMEM;
2137 	skb_copy_decrypted(buff, skb);
2138 	mptcp_skb_ext_copy(buff, skb);
2139 
2140 	sk_wmem_queued_add(sk, buff->truesize);
2141 	sk_mem_charge(sk, buff->truesize);
2142 	buff->truesize += nlen;
2143 	skb->truesize -= nlen;
2144 
2145 	/* Correct the sequence numbers. */
2146 	TCP_SKB_CB(buff)->seq = TCP_SKB_CB(skb)->seq + len;
2147 	TCP_SKB_CB(buff)->end_seq = TCP_SKB_CB(skb)->end_seq;
2148 	TCP_SKB_CB(skb)->end_seq = TCP_SKB_CB(buff)->seq;
2149 
2150 	/* PSH and FIN should only be set in the second packet. */
2151 	flags = TCP_SKB_CB(skb)->tcp_flags;
2152 	TCP_SKB_CB(skb)->tcp_flags = flags & ~(TCPHDR_FIN | TCPHDR_PSH);
2153 	TCP_SKB_CB(buff)->tcp_flags = flags;
2154 
2155 	tcp_skb_fragment_eor(skb, buff);
2156 
2157 	skb_split(skb, buff, len);
2158 	tcp_fragment_tstamp(skb, buff);
2159 
2160 	/* Fix up tso_factor for both original and new SKB.  */
2161 	tcp_set_skb_tso_segs(skb, mss_now);
2162 	tcp_set_skb_tso_segs(buff, mss_now);
2163 
2164 	/* Link BUFF into the send queue. */
2165 	__skb_header_release(buff);
2166 	tcp_insert_write_queue_after(skb, buff, sk, TCP_FRAG_IN_WRITE_QUEUE);
2167 
2168 	return 0;
2169 }
2170 
2171 /* Try to defer sending, if possible, in order to minimize the amount
2172  * of TSO splitting we do.  View it as a kind of TSO Nagle test.
2173  *
2174  * This algorithm is from John Heffner.
2175  */
2176 static bool tcp_tso_should_defer(struct sock *sk, struct sk_buff *skb,
2177 				 bool *is_cwnd_limited,
2178 				 bool *is_rwnd_limited,
2179 				 u32 max_segs)
2180 {
2181 	const struct inet_connection_sock *icsk = inet_csk(sk);
2182 	u32 send_win, cong_win, limit, in_flight;
2183 	struct tcp_sock *tp = tcp_sk(sk);
2184 	struct sk_buff *head;
2185 	int win_divisor;
2186 	s64 delta;
2187 
2188 	if (icsk->icsk_ca_state >= TCP_CA_Recovery)
2189 		goto send_now;
2190 
2191 	/* Avoid bursty behavior by allowing defer
2192 	 * only if the last write was recent (1 ms).
2193 	 * Note that tp->tcp_wstamp_ns can be in the future if we have
2194 	 * packets waiting in a qdisc or device for EDT delivery.
2195 	 */
2196 	delta = tp->tcp_clock_cache - tp->tcp_wstamp_ns - NSEC_PER_MSEC;
2197 	if (delta > 0)
2198 		goto send_now;
2199 
2200 	in_flight = tcp_packets_in_flight(tp);
2201 
2202 	BUG_ON(tcp_skb_pcount(skb) <= 1);
2203 	BUG_ON(tcp_snd_cwnd(tp) <= in_flight);
2204 
2205 	send_win = tcp_wnd_end(tp) - TCP_SKB_CB(skb)->seq;
2206 
2207 	/* From in_flight test above, we know that cwnd > in_flight.  */
2208 	cong_win = (tcp_snd_cwnd(tp) - in_flight) * tp->mss_cache;
2209 
2210 	limit = min(send_win, cong_win);
2211 
2212 	/* If a full-sized TSO skb can be sent, do it. */
2213 	if (limit >= max_segs * tp->mss_cache)
2214 		goto send_now;
2215 
2216 	/* Middle in queue won't get any more data, full sendable already? */
2217 	if ((skb != tcp_write_queue_tail(sk)) && (limit >= skb->len))
2218 		goto send_now;
2219 
2220 	win_divisor = READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_tso_win_divisor);
2221 	if (win_divisor) {
2222 		u32 chunk = min(tp->snd_wnd, tcp_snd_cwnd(tp) * tp->mss_cache);
2223 
2224 		/* If at least some fraction of a window is available,
2225 		 * just use it.
2226 		 */
2227 		chunk /= win_divisor;
2228 		if (limit >= chunk)
2229 			goto send_now;
2230 	} else {
2231 		/* Different approach, try not to defer past a single
2232 		 * ACK.  Receiver should ACK every other full sized
2233 		 * frame, so if we have space for more than 3 frames
2234 		 * then send now.
2235 		 */
2236 		if (limit > tcp_max_tso_deferred_mss(tp) * tp->mss_cache)
2237 			goto send_now;
2238 	}
2239 
2240 	/* TODO : use tsorted_sent_queue ? */
2241 	head = tcp_rtx_queue_head(sk);
2242 	if (!head)
2243 		goto send_now;
2244 	delta = tp->tcp_clock_cache - head->tstamp;
2245 	/* If next ACK is likely to come too late (half srtt), do not defer */
2246 	if ((s64)(delta - (u64)NSEC_PER_USEC * (tp->srtt_us >> 4)) < 0)
2247 		goto send_now;
2248 
2249 	/* Ok, it looks like it is advisable to defer.
2250 	 * Three cases are tracked :
2251 	 * 1) We are cwnd-limited
2252 	 * 2) We are rwnd-limited
2253 	 * 3) We are application limited.
2254 	 */
2255 	if (cong_win < send_win) {
2256 		if (cong_win <= skb->len) {
2257 			*is_cwnd_limited = true;
2258 			return true;
2259 		}
2260 	} else {
2261 		if (send_win <= skb->len) {
2262 			*is_rwnd_limited = true;
2263 			return true;
2264 		}
2265 	}
2266 
2267 	/* If this packet won't get more data, do not wait. */
2268 	if ((TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN) ||
2269 	    TCP_SKB_CB(skb)->eor)
2270 		goto send_now;
2271 
2272 	return true;
2273 
2274 send_now:
2275 	return false;
2276 }
2277 
2278 static inline void tcp_mtu_check_reprobe(struct sock *sk)
2279 {
2280 	struct inet_connection_sock *icsk = inet_csk(sk);
2281 	struct tcp_sock *tp = tcp_sk(sk);
2282 	struct net *net = sock_net(sk);
2283 	u32 interval;
2284 	s32 delta;
2285 
2286 	interval = READ_ONCE(net->ipv4.sysctl_tcp_probe_interval);
2287 	delta = tcp_jiffies32 - icsk->icsk_mtup.probe_timestamp;
2288 	if (unlikely(delta >= interval * HZ)) {
2289 		int mss = tcp_current_mss(sk);
2290 
2291 		/* Update current search range */
2292 		icsk->icsk_mtup.probe_size = 0;
2293 		icsk->icsk_mtup.search_high = tp->rx_opt.mss_clamp +
2294 			sizeof(struct tcphdr) +
2295 			icsk->icsk_af_ops->net_header_len;
2296 		icsk->icsk_mtup.search_low = tcp_mss_to_mtu(sk, mss);
2297 
2298 		/* Update probe time stamp */
2299 		icsk->icsk_mtup.probe_timestamp = tcp_jiffies32;
2300 	}
2301 }
2302 
2303 static bool tcp_can_coalesce_send_queue_head(struct sock *sk, int len)
2304 {
2305 	struct sk_buff *skb, *next;
2306 
2307 	skb = tcp_send_head(sk);
2308 	tcp_for_write_queue_from_safe(skb, next, sk) {
2309 		if (len <= skb->len)
2310 			break;
2311 
2312 		if (unlikely(TCP_SKB_CB(skb)->eor) ||
2313 		    tcp_has_tx_tstamp(skb) ||
2314 		    !skb_pure_zcopy_same(skb, next))
2315 			return false;
2316 
2317 		len -= skb->len;
2318 	}
2319 
2320 	return true;
2321 }
2322 
2323 /* Create a new MTU probe if we are ready.
2324  * MTU probe is regularly attempting to increase the path MTU by
2325  * deliberately sending larger packets.  This discovers routing
2326  * changes resulting in larger path MTUs.
2327  *
2328  * Returns 0 if we should wait to probe (no cwnd available),
2329  *         1 if a probe was sent,
2330  *         -1 otherwise
2331  */
2332 static int tcp_mtu_probe(struct sock *sk)
2333 {
2334 	struct inet_connection_sock *icsk = inet_csk(sk);
2335 	struct tcp_sock *tp = tcp_sk(sk);
2336 	struct sk_buff *skb, *nskb, *next;
2337 	struct net *net = sock_net(sk);
2338 	int probe_size;
2339 	int size_needed;
2340 	int copy, len;
2341 	int mss_now;
2342 	int interval;
2343 
2344 	/* Not currently probing/verifying,
2345 	 * not in recovery,
2346 	 * have enough cwnd, and
2347 	 * not SACKing (the variable headers throw things off)
2348 	 */
2349 	if (likely(!icsk->icsk_mtup.enabled ||
2350 		   icsk->icsk_mtup.probe_size ||
2351 		   inet_csk(sk)->icsk_ca_state != TCP_CA_Open ||
2352 		   tcp_snd_cwnd(tp) < 11 ||
2353 		   tp->rx_opt.num_sacks || tp->rx_opt.dsack))
2354 		return -1;
2355 
2356 	/* Use binary search for probe_size between tcp_mss_base,
2357 	 * and current mss_clamp. if (search_high - search_low)
2358 	 * smaller than a threshold, backoff from probing.
2359 	 */
2360 	mss_now = tcp_current_mss(sk);
2361 	probe_size = tcp_mtu_to_mss(sk, (icsk->icsk_mtup.search_high +
2362 				    icsk->icsk_mtup.search_low) >> 1);
2363 	size_needed = probe_size + (tp->reordering + 1) * tp->mss_cache;
2364 	interval = icsk->icsk_mtup.search_high - icsk->icsk_mtup.search_low;
2365 	/* When misfortune happens, we are reprobing actively,
2366 	 * and then reprobe timer has expired. We stick with current
2367 	 * probing process by not resetting search range to its orignal.
2368 	 */
2369 	if (probe_size > tcp_mtu_to_mss(sk, icsk->icsk_mtup.search_high) ||
2370 	    interval < READ_ONCE(net->ipv4.sysctl_tcp_probe_threshold)) {
2371 		/* Check whether enough time has elaplased for
2372 		 * another round of probing.
2373 		 */
2374 		tcp_mtu_check_reprobe(sk);
2375 		return -1;
2376 	}
2377 
2378 	/* Have enough data in the send queue to probe? */
2379 	if (tp->write_seq - tp->snd_nxt < size_needed)
2380 		return -1;
2381 
2382 	if (tp->snd_wnd < size_needed)
2383 		return -1;
2384 	if (after(tp->snd_nxt + size_needed, tcp_wnd_end(tp)))
2385 		return 0;
2386 
2387 	/* Do we need to wait to drain cwnd? With none in flight, don't stall */
2388 	if (tcp_packets_in_flight(tp) + 2 > tcp_snd_cwnd(tp)) {
2389 		if (!tcp_packets_in_flight(tp))
2390 			return -1;
2391 		else
2392 			return 0;
2393 	}
2394 
2395 	if (!tcp_can_coalesce_send_queue_head(sk, probe_size))
2396 		return -1;
2397 
2398 	/* We're allowed to probe.  Build it now. */
2399 	nskb = tcp_stream_alloc_skb(sk, probe_size, GFP_ATOMIC, false);
2400 	if (!nskb)
2401 		return -1;
2402 	sk_wmem_queued_add(sk, nskb->truesize);
2403 	sk_mem_charge(sk, nskb->truesize);
2404 
2405 	skb = tcp_send_head(sk);
2406 	skb_copy_decrypted(nskb, skb);
2407 	mptcp_skb_ext_copy(nskb, skb);
2408 
2409 	TCP_SKB_CB(nskb)->seq = TCP_SKB_CB(skb)->seq;
2410 	TCP_SKB_CB(nskb)->end_seq = TCP_SKB_CB(skb)->seq + probe_size;
2411 	TCP_SKB_CB(nskb)->tcp_flags = TCPHDR_ACK;
2412 
2413 	tcp_insert_write_queue_before(nskb, skb, sk);
2414 	tcp_highest_sack_replace(sk, skb, nskb);
2415 
2416 	len = 0;
2417 	tcp_for_write_queue_from_safe(skb, next, sk) {
2418 		copy = min_t(int, skb->len, probe_size - len);
2419 		skb_copy_bits(skb, 0, skb_put(nskb, copy), copy);
2420 
2421 		if (skb->len <= copy) {
2422 			/* We've eaten all the data from this skb.
2423 			 * Throw it away. */
2424 			TCP_SKB_CB(nskb)->tcp_flags |= TCP_SKB_CB(skb)->tcp_flags;
2425 			/* If this is the last SKB we copy and eor is set
2426 			 * we need to propagate it to the new skb.
2427 			 */
2428 			TCP_SKB_CB(nskb)->eor = TCP_SKB_CB(skb)->eor;
2429 			tcp_skb_collapse_tstamp(nskb, skb);
2430 			tcp_unlink_write_queue(skb, sk);
2431 			tcp_wmem_free_skb(sk, skb);
2432 		} else {
2433 			TCP_SKB_CB(nskb)->tcp_flags |= TCP_SKB_CB(skb)->tcp_flags &
2434 						   ~(TCPHDR_FIN|TCPHDR_PSH);
2435 			if (!skb_shinfo(skb)->nr_frags) {
2436 				skb_pull(skb, copy);
2437 			} else {
2438 				__pskb_trim_head(skb, copy);
2439 				tcp_set_skb_tso_segs(skb, mss_now);
2440 			}
2441 			TCP_SKB_CB(skb)->seq += copy;
2442 		}
2443 
2444 		len += copy;
2445 
2446 		if (len >= probe_size)
2447 			break;
2448 	}
2449 	tcp_init_tso_segs(nskb, nskb->len);
2450 
2451 	/* We're ready to send.  If this fails, the probe will
2452 	 * be resegmented into mss-sized pieces by tcp_write_xmit().
2453 	 */
2454 	if (!tcp_transmit_skb(sk, nskb, 1, GFP_ATOMIC)) {
2455 		/* Decrement cwnd here because we are sending
2456 		 * effectively two packets. */
2457 		tcp_snd_cwnd_set(tp, tcp_snd_cwnd(tp) - 1);
2458 		tcp_event_new_data_sent(sk, nskb);
2459 
2460 		icsk->icsk_mtup.probe_size = tcp_mss_to_mtu(sk, nskb->len);
2461 		tp->mtu_probe.probe_seq_start = TCP_SKB_CB(nskb)->seq;
2462 		tp->mtu_probe.probe_seq_end = TCP_SKB_CB(nskb)->end_seq;
2463 
2464 		return 1;
2465 	}
2466 
2467 	return -1;
2468 }
2469 
2470 static bool tcp_pacing_check(struct sock *sk)
2471 {
2472 	struct tcp_sock *tp = tcp_sk(sk);
2473 
2474 	if (!tcp_needs_internal_pacing(sk))
2475 		return false;
2476 
2477 	if (tp->tcp_wstamp_ns <= tp->tcp_clock_cache)
2478 		return false;
2479 
2480 	if (!hrtimer_is_queued(&tp->pacing_timer)) {
2481 		hrtimer_start(&tp->pacing_timer,
2482 			      ns_to_ktime(tp->tcp_wstamp_ns),
2483 			      HRTIMER_MODE_ABS_PINNED_SOFT);
2484 		sock_hold(sk);
2485 	}
2486 	return true;
2487 }
2488 
2489 /* TCP Small Queues :
2490  * Control number of packets in qdisc/devices to two packets / or ~1 ms.
2491  * (These limits are doubled for retransmits)
2492  * This allows for :
2493  *  - better RTT estimation and ACK scheduling
2494  *  - faster recovery
2495  *  - high rates
2496  * Alas, some drivers / subsystems require a fair amount
2497  * of queued bytes to ensure line rate.
2498  * One example is wifi aggregation (802.11 AMPDU)
2499  */
2500 static bool tcp_small_queue_check(struct sock *sk, const struct sk_buff *skb,
2501 				  unsigned int factor)
2502 {
2503 	unsigned long limit;
2504 
2505 	limit = max_t(unsigned long,
2506 		      2 * skb->truesize,
2507 		      sk->sk_pacing_rate >> READ_ONCE(sk->sk_pacing_shift));
2508 	if (sk->sk_pacing_status == SK_PACING_NONE)
2509 		limit = min_t(unsigned long, limit,
2510 			      sock_net(sk)->ipv4.sysctl_tcp_limit_output_bytes);
2511 	limit <<= factor;
2512 
2513 	if (static_branch_unlikely(&tcp_tx_delay_enabled) &&
2514 	    tcp_sk(sk)->tcp_tx_delay) {
2515 		u64 extra_bytes = (u64)sk->sk_pacing_rate * tcp_sk(sk)->tcp_tx_delay;
2516 
2517 		/* TSQ is based on skb truesize sum (sk_wmem_alloc), so we
2518 		 * approximate our needs assuming an ~100% skb->truesize overhead.
2519 		 * USEC_PER_SEC is approximated by 2^20.
2520 		 * do_div(extra_bytes, USEC_PER_SEC/2) is replaced by a right shift.
2521 		 */
2522 		extra_bytes >>= (20 - 1);
2523 		limit += extra_bytes;
2524 	}
2525 	if (refcount_read(&sk->sk_wmem_alloc) > limit) {
2526 		/* Always send skb if rtx queue is empty.
2527 		 * No need to wait for TX completion to call us back,
2528 		 * after softirq/tasklet schedule.
2529 		 * This helps when TX completions are delayed too much.
2530 		 */
2531 		if (tcp_rtx_queue_empty(sk))
2532 			return false;
2533 
2534 		set_bit(TSQ_THROTTLED, &sk->sk_tsq_flags);
2535 		/* It is possible TX completion already happened
2536 		 * before we set TSQ_THROTTLED, so we must
2537 		 * test again the condition.
2538 		 */
2539 		smp_mb__after_atomic();
2540 		if (refcount_read(&sk->sk_wmem_alloc) > limit)
2541 			return true;
2542 	}
2543 	return false;
2544 }
2545 
2546 static void tcp_chrono_set(struct tcp_sock *tp, const enum tcp_chrono new)
2547 {
2548 	const u32 now = tcp_jiffies32;
2549 	enum tcp_chrono old = tp->chrono_type;
2550 
2551 	if (old > TCP_CHRONO_UNSPEC)
2552 		tp->chrono_stat[old - 1] += now - tp->chrono_start;
2553 	tp->chrono_start = now;
2554 	tp->chrono_type = new;
2555 }
2556 
2557 void tcp_chrono_start(struct sock *sk, const enum tcp_chrono type)
2558 {
2559 	struct tcp_sock *tp = tcp_sk(sk);
2560 
2561 	/* If there are multiple conditions worthy of tracking in a
2562 	 * chronograph then the highest priority enum takes precedence
2563 	 * over the other conditions. So that if something "more interesting"
2564 	 * starts happening, stop the previous chrono and start a new one.
2565 	 */
2566 	if (type > tp->chrono_type)
2567 		tcp_chrono_set(tp, type);
2568 }
2569 
2570 void tcp_chrono_stop(struct sock *sk, const enum tcp_chrono type)
2571 {
2572 	struct tcp_sock *tp = tcp_sk(sk);
2573 
2574 
2575 	/* There are multiple conditions worthy of tracking in a
2576 	 * chronograph, so that the highest priority enum takes
2577 	 * precedence over the other conditions (see tcp_chrono_start).
2578 	 * If a condition stops, we only stop chrono tracking if
2579 	 * it's the "most interesting" or current chrono we are
2580 	 * tracking and starts busy chrono if we have pending data.
2581 	 */
2582 	if (tcp_rtx_and_write_queues_empty(sk))
2583 		tcp_chrono_set(tp, TCP_CHRONO_UNSPEC);
2584 	else if (type == tp->chrono_type)
2585 		tcp_chrono_set(tp, TCP_CHRONO_BUSY);
2586 }
2587 
2588 /* This routine writes packets to the network.  It advances the
2589  * send_head.  This happens as incoming acks open up the remote
2590  * window for us.
2591  *
2592  * LARGESEND note: !tcp_urg_mode is overkill, only frames between
2593  * snd_up-64k-mss .. snd_up cannot be large. However, taking into
2594  * account rare use of URG, this is not a big flaw.
2595  *
2596  * Send at most one packet when push_one > 0. Temporarily ignore
2597  * cwnd limit to force at most one packet out when push_one == 2.
2598 
2599  * Returns true, if no segments are in flight and we have queued segments,
2600  * but cannot send anything now because of SWS or another problem.
2601  */
2602 static bool tcp_write_xmit(struct sock *sk, unsigned int mss_now, int nonagle,
2603 			   int push_one, gfp_t gfp)
2604 {
2605 	struct tcp_sock *tp = tcp_sk(sk);
2606 	struct sk_buff *skb;
2607 	unsigned int tso_segs, sent_pkts;
2608 	int cwnd_quota;
2609 	int result;
2610 	bool is_cwnd_limited = false, is_rwnd_limited = false;
2611 	u32 max_segs;
2612 
2613 	sent_pkts = 0;
2614 
2615 	tcp_mstamp_refresh(tp);
2616 	if (!push_one) {
2617 		/* Do MTU probing. */
2618 		result = tcp_mtu_probe(sk);
2619 		if (!result) {
2620 			return false;
2621 		} else if (result > 0) {
2622 			sent_pkts = 1;
2623 		}
2624 	}
2625 
2626 	max_segs = tcp_tso_segs(sk, mss_now);
2627 	while ((skb = tcp_send_head(sk))) {
2628 		unsigned int limit;
2629 
2630 		if (unlikely(tp->repair) && tp->repair_queue == TCP_SEND_QUEUE) {
2631 			/* "skb_mstamp_ns" is used as a start point for the retransmit timer */
2632 			tp->tcp_wstamp_ns = tp->tcp_clock_cache;
2633 			skb_set_delivery_time(skb, tp->tcp_wstamp_ns, true);
2634 			list_move_tail(&skb->tcp_tsorted_anchor, &tp->tsorted_sent_queue);
2635 			tcp_init_tso_segs(skb, mss_now);
2636 			goto repair; /* Skip network transmission */
2637 		}
2638 
2639 		if (tcp_pacing_check(sk))
2640 			break;
2641 
2642 		tso_segs = tcp_init_tso_segs(skb, mss_now);
2643 		BUG_ON(!tso_segs);
2644 
2645 		cwnd_quota = tcp_cwnd_test(tp, skb);
2646 		if (!cwnd_quota) {
2647 			if (push_one == 2)
2648 				/* Force out a loss probe pkt. */
2649 				cwnd_quota = 1;
2650 			else
2651 				break;
2652 		}
2653 
2654 		if (unlikely(!tcp_snd_wnd_test(tp, skb, mss_now))) {
2655 			is_rwnd_limited = true;
2656 			break;
2657 		}
2658 
2659 		if (tso_segs == 1) {
2660 			if (unlikely(!tcp_nagle_test(tp, skb, mss_now,
2661 						     (tcp_skb_is_last(sk, skb) ?
2662 						      nonagle : TCP_NAGLE_PUSH))))
2663 				break;
2664 		} else {
2665 			if (!push_one &&
2666 			    tcp_tso_should_defer(sk, skb, &is_cwnd_limited,
2667 						 &is_rwnd_limited, max_segs))
2668 				break;
2669 		}
2670 
2671 		limit = mss_now;
2672 		if (tso_segs > 1 && !tcp_urg_mode(tp))
2673 			limit = tcp_mss_split_point(sk, skb, mss_now,
2674 						    min_t(unsigned int,
2675 							  cwnd_quota,
2676 							  max_segs),
2677 						    nonagle);
2678 
2679 		if (skb->len > limit &&
2680 		    unlikely(tso_fragment(sk, skb, limit, mss_now, gfp)))
2681 			break;
2682 
2683 		if (tcp_small_queue_check(sk, skb, 0))
2684 			break;
2685 
2686 		/* Argh, we hit an empty skb(), presumably a thread
2687 		 * is sleeping in sendmsg()/sk_stream_wait_memory().
2688 		 * We do not want to send a pure-ack packet and have
2689 		 * a strange looking rtx queue with empty packet(s).
2690 		 */
2691 		if (TCP_SKB_CB(skb)->end_seq == TCP_SKB_CB(skb)->seq)
2692 			break;
2693 
2694 		if (unlikely(tcp_transmit_skb(sk, skb, 1, gfp)))
2695 			break;
2696 
2697 repair:
2698 		/* Advance the send_head.  This one is sent out.
2699 		 * This call will increment packets_out.
2700 		 */
2701 		tcp_event_new_data_sent(sk, skb);
2702 
2703 		tcp_minshall_update(tp, mss_now, skb);
2704 		sent_pkts += tcp_skb_pcount(skb);
2705 
2706 		if (push_one)
2707 			break;
2708 	}
2709 
2710 	if (is_rwnd_limited)
2711 		tcp_chrono_start(sk, TCP_CHRONO_RWND_LIMITED);
2712 	else
2713 		tcp_chrono_stop(sk, TCP_CHRONO_RWND_LIMITED);
2714 
2715 	is_cwnd_limited |= (tcp_packets_in_flight(tp) >= tcp_snd_cwnd(tp));
2716 	if (likely(sent_pkts || is_cwnd_limited))
2717 		tcp_cwnd_validate(sk, is_cwnd_limited);
2718 
2719 	if (likely(sent_pkts)) {
2720 		if (tcp_in_cwnd_reduction(sk))
2721 			tp->prr_out += sent_pkts;
2722 
2723 		/* Send one loss probe per tail loss episode. */
2724 		if (push_one != 2)
2725 			tcp_schedule_loss_probe(sk, false);
2726 		return false;
2727 	}
2728 	return !tp->packets_out && !tcp_write_queue_empty(sk);
2729 }
2730 
2731 bool tcp_schedule_loss_probe(struct sock *sk, bool advancing_rto)
2732 {
2733 	struct inet_connection_sock *icsk = inet_csk(sk);
2734 	struct tcp_sock *tp = tcp_sk(sk);
2735 	u32 timeout, rto_delta_us;
2736 	int early_retrans;
2737 
2738 	/* Don't do any loss probe on a Fast Open connection before 3WHS
2739 	 * finishes.
2740 	 */
2741 	if (rcu_access_pointer(tp->fastopen_rsk))
2742 		return false;
2743 
2744 	early_retrans = READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_early_retrans);
2745 	/* Schedule a loss probe in 2*RTT for SACK capable connections
2746 	 * not in loss recovery, that are either limited by cwnd or application.
2747 	 */
2748 	if ((early_retrans != 3 && early_retrans != 4) ||
2749 	    !tp->packets_out || !tcp_is_sack(tp) ||
2750 	    (icsk->icsk_ca_state != TCP_CA_Open &&
2751 	     icsk->icsk_ca_state != TCP_CA_CWR))
2752 		return false;
2753 
2754 	/* Probe timeout is 2*rtt. Add minimum RTO to account
2755 	 * for delayed ack when there's one outstanding packet. If no RTT
2756 	 * sample is available then probe after TCP_TIMEOUT_INIT.
2757 	 */
2758 	if (tp->srtt_us) {
2759 		timeout = usecs_to_jiffies(tp->srtt_us >> 2);
2760 		if (tp->packets_out == 1)
2761 			timeout += TCP_RTO_MIN;
2762 		else
2763 			timeout += TCP_TIMEOUT_MIN;
2764 	} else {
2765 		timeout = TCP_TIMEOUT_INIT;
2766 	}
2767 
2768 	/* If the RTO formula yields an earlier time, then use that time. */
2769 	rto_delta_us = advancing_rto ?
2770 			jiffies_to_usecs(inet_csk(sk)->icsk_rto) :
2771 			tcp_rto_delta_us(sk);  /* How far in future is RTO? */
2772 	if (rto_delta_us > 0)
2773 		timeout = min_t(u32, timeout, usecs_to_jiffies(rto_delta_us));
2774 
2775 	tcp_reset_xmit_timer(sk, ICSK_TIME_LOSS_PROBE, timeout, TCP_RTO_MAX);
2776 	return true;
2777 }
2778 
2779 /* Thanks to skb fast clones, we can detect if a prior transmit of
2780  * a packet is still in a qdisc or driver queue.
2781  * In this case, there is very little point doing a retransmit !
2782  */
2783 static bool skb_still_in_host_queue(struct sock *sk,
2784 				    const struct sk_buff *skb)
2785 {
2786 	if (unlikely(skb_fclone_busy(sk, skb))) {
2787 		set_bit(TSQ_THROTTLED, &sk->sk_tsq_flags);
2788 		smp_mb__after_atomic();
2789 		if (skb_fclone_busy(sk, skb)) {
2790 			NET_INC_STATS(sock_net(sk),
2791 				      LINUX_MIB_TCPSPURIOUS_RTX_HOSTQUEUES);
2792 			return true;
2793 		}
2794 	}
2795 	return false;
2796 }
2797 
2798 /* When probe timeout (PTO) fires, try send a new segment if possible, else
2799  * retransmit the last segment.
2800  */
2801 void tcp_send_loss_probe(struct sock *sk)
2802 {
2803 	struct tcp_sock *tp = tcp_sk(sk);
2804 	struct sk_buff *skb;
2805 	int pcount;
2806 	int mss = tcp_current_mss(sk);
2807 
2808 	/* At most one outstanding TLP */
2809 	if (tp->tlp_high_seq)
2810 		goto rearm_timer;
2811 
2812 	tp->tlp_retrans = 0;
2813 	skb = tcp_send_head(sk);
2814 	if (skb && tcp_snd_wnd_test(tp, skb, mss)) {
2815 		pcount = tp->packets_out;
2816 		tcp_write_xmit(sk, mss, TCP_NAGLE_OFF, 2, GFP_ATOMIC);
2817 		if (tp->packets_out > pcount)
2818 			goto probe_sent;
2819 		goto rearm_timer;
2820 	}
2821 	skb = skb_rb_last(&sk->tcp_rtx_queue);
2822 	if (unlikely(!skb)) {
2823 		WARN_ONCE(tp->packets_out,
2824 			  "invalid inflight: %u state %u cwnd %u mss %d\n",
2825 			  tp->packets_out, sk->sk_state, tcp_snd_cwnd(tp), mss);
2826 		inet_csk(sk)->icsk_pending = 0;
2827 		return;
2828 	}
2829 
2830 	if (skb_still_in_host_queue(sk, skb))
2831 		goto rearm_timer;
2832 
2833 	pcount = tcp_skb_pcount(skb);
2834 	if (WARN_ON(!pcount))
2835 		goto rearm_timer;
2836 
2837 	if ((pcount > 1) && (skb->len > (pcount - 1) * mss)) {
2838 		if (unlikely(tcp_fragment(sk, TCP_FRAG_IN_RTX_QUEUE, skb,
2839 					  (pcount - 1) * mss, mss,
2840 					  GFP_ATOMIC)))
2841 			goto rearm_timer;
2842 		skb = skb_rb_next(skb);
2843 	}
2844 
2845 	if (WARN_ON(!skb || !tcp_skb_pcount(skb)))
2846 		goto rearm_timer;
2847 
2848 	if (__tcp_retransmit_skb(sk, skb, 1))
2849 		goto rearm_timer;
2850 
2851 	tp->tlp_retrans = 1;
2852 
2853 probe_sent:
2854 	/* Record snd_nxt for loss detection. */
2855 	tp->tlp_high_seq = tp->snd_nxt;
2856 
2857 	NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPLOSSPROBES);
2858 	/* Reset s.t. tcp_rearm_rto will restart timer from now */
2859 	inet_csk(sk)->icsk_pending = 0;
2860 rearm_timer:
2861 	tcp_rearm_rto(sk);
2862 }
2863 
2864 /* Push out any pending frames which were held back due to
2865  * TCP_CORK or attempt at coalescing tiny packets.
2866  * The socket must be locked by the caller.
2867  */
2868 void __tcp_push_pending_frames(struct sock *sk, unsigned int cur_mss,
2869 			       int nonagle)
2870 {
2871 	/* If we are closed, the bytes will have to remain here.
2872 	 * In time closedown will finish, we empty the write queue and
2873 	 * all will be happy.
2874 	 */
2875 	if (unlikely(sk->sk_state == TCP_CLOSE))
2876 		return;
2877 
2878 	if (tcp_write_xmit(sk, cur_mss, nonagle, 0,
2879 			   sk_gfp_mask(sk, GFP_ATOMIC)))
2880 		tcp_check_probe_timer(sk);
2881 }
2882 
2883 /* Send _single_ skb sitting at the send head. This function requires
2884  * true push pending frames to setup probe timer etc.
2885  */
2886 void tcp_push_one(struct sock *sk, unsigned int mss_now)
2887 {
2888 	struct sk_buff *skb = tcp_send_head(sk);
2889 
2890 	BUG_ON(!skb || skb->len < mss_now);
2891 
2892 	tcp_write_xmit(sk, mss_now, TCP_NAGLE_PUSH, 1, sk->sk_allocation);
2893 }
2894 
2895 /* This function returns the amount that we can raise the
2896  * usable window based on the following constraints
2897  *
2898  * 1. The window can never be shrunk once it is offered (RFC 793)
2899  * 2. We limit memory per socket
2900  *
2901  * RFC 1122:
2902  * "the suggested [SWS] avoidance algorithm for the receiver is to keep
2903  *  RECV.NEXT + RCV.WIN fixed until:
2904  *  RCV.BUFF - RCV.USER - RCV.WINDOW >= min(1/2 RCV.BUFF, MSS)"
2905  *
2906  * i.e. don't raise the right edge of the window until you can raise
2907  * it at least MSS bytes.
2908  *
2909  * Unfortunately, the recommended algorithm breaks header prediction,
2910  * since header prediction assumes th->window stays fixed.
2911  *
2912  * Strictly speaking, keeping th->window fixed violates the receiver
2913  * side SWS prevention criteria. The problem is that under this rule
2914  * a stream of single byte packets will cause the right side of the
2915  * window to always advance by a single byte.
2916  *
2917  * Of course, if the sender implements sender side SWS prevention
2918  * then this will not be a problem.
2919  *
2920  * BSD seems to make the following compromise:
2921  *
2922  *	If the free space is less than the 1/4 of the maximum
2923  *	space available and the free space is less than 1/2 mss,
2924  *	then set the window to 0.
2925  *	[ Actually, bsd uses MSS and 1/4 of maximal _window_ ]
2926  *	Otherwise, just prevent the window from shrinking
2927  *	and from being larger than the largest representable value.
2928  *
2929  * This prevents incremental opening of the window in the regime
2930  * where TCP is limited by the speed of the reader side taking
2931  * data out of the TCP receive queue. It does nothing about
2932  * those cases where the window is constrained on the sender side
2933  * because the pipeline is full.
2934  *
2935  * BSD also seems to "accidentally" limit itself to windows that are a
2936  * multiple of MSS, at least until the free space gets quite small.
2937  * This would appear to be a side effect of the mbuf implementation.
2938  * Combining these two algorithms results in the observed behavior
2939  * of having a fixed window size at almost all times.
2940  *
2941  * Below we obtain similar behavior by forcing the offered window to
2942  * a multiple of the mss when it is feasible to do so.
2943  *
2944  * Note, we don't "adjust" for TIMESTAMP or SACK option bytes.
2945  * Regular options like TIMESTAMP are taken into account.
2946  */
2947 u32 __tcp_select_window(struct sock *sk)
2948 {
2949 	struct inet_connection_sock *icsk = inet_csk(sk);
2950 	struct tcp_sock *tp = tcp_sk(sk);
2951 	/* MSS for the peer's data.  Previous versions used mss_clamp
2952 	 * here.  I don't know if the value based on our guesses
2953 	 * of peer's MSS is better for the performance.  It's more correct
2954 	 * but may be worse for the performance because of rcv_mss
2955 	 * fluctuations.  --SAW  1998/11/1
2956 	 */
2957 	int mss = icsk->icsk_ack.rcv_mss;
2958 	int free_space = tcp_space(sk);
2959 	int allowed_space = tcp_full_space(sk);
2960 	int full_space, window;
2961 
2962 	if (sk_is_mptcp(sk))
2963 		mptcp_space(sk, &free_space, &allowed_space);
2964 
2965 	full_space = min_t(int, tp->window_clamp, allowed_space);
2966 
2967 	if (unlikely(mss > full_space)) {
2968 		mss = full_space;
2969 		if (mss <= 0)
2970 			return 0;
2971 	}
2972 	if (free_space < (full_space >> 1)) {
2973 		icsk->icsk_ack.quick = 0;
2974 
2975 		if (tcp_under_memory_pressure(sk))
2976 			tcp_adjust_rcv_ssthresh(sk);
2977 
2978 		/* free_space might become our new window, make sure we don't
2979 		 * increase it due to wscale.
2980 		 */
2981 		free_space = round_down(free_space, 1 << tp->rx_opt.rcv_wscale);
2982 
2983 		/* if free space is less than mss estimate, or is below 1/16th
2984 		 * of the maximum allowed, try to move to zero-window, else
2985 		 * tcp_clamp_window() will grow rcv buf up to tcp_rmem[2], and
2986 		 * new incoming data is dropped due to memory limits.
2987 		 * With large window, mss test triggers way too late in order
2988 		 * to announce zero window in time before rmem limit kicks in.
2989 		 */
2990 		if (free_space < (allowed_space >> 4) || free_space < mss)
2991 			return 0;
2992 	}
2993 
2994 	if (free_space > tp->rcv_ssthresh)
2995 		free_space = tp->rcv_ssthresh;
2996 
2997 	/* Don't do rounding if we are using window scaling, since the
2998 	 * scaled window will not line up with the MSS boundary anyway.
2999 	 */
3000 	if (tp->rx_opt.rcv_wscale) {
3001 		window = free_space;
3002 
3003 		/* Advertise enough space so that it won't get scaled away.
3004 		 * Import case: prevent zero window announcement if
3005 		 * 1<<rcv_wscale > mss.
3006 		 */
3007 		window = ALIGN(window, (1 << tp->rx_opt.rcv_wscale));
3008 	} else {
3009 		window = tp->rcv_wnd;
3010 		/* Get the largest window that is a nice multiple of mss.
3011 		 * Window clamp already applied above.
3012 		 * If our current window offering is within 1 mss of the
3013 		 * free space we just keep it. This prevents the divide
3014 		 * and multiply from happening most of the time.
3015 		 * We also don't do any window rounding when the free space
3016 		 * is too small.
3017 		 */
3018 		if (window <= free_space - mss || window > free_space)
3019 			window = rounddown(free_space, mss);
3020 		else if (mss == full_space &&
3021 			 free_space > window + (full_space >> 1))
3022 			window = free_space;
3023 	}
3024 
3025 	return window;
3026 }
3027 
3028 void tcp_skb_collapse_tstamp(struct sk_buff *skb,
3029 			     const struct sk_buff *next_skb)
3030 {
3031 	if (unlikely(tcp_has_tx_tstamp(next_skb))) {
3032 		const struct skb_shared_info *next_shinfo =
3033 			skb_shinfo(next_skb);
3034 		struct skb_shared_info *shinfo = skb_shinfo(skb);
3035 
3036 		shinfo->tx_flags |= next_shinfo->tx_flags & SKBTX_ANY_TSTAMP;
3037 		shinfo->tskey = next_shinfo->tskey;
3038 		TCP_SKB_CB(skb)->txstamp_ack |=
3039 			TCP_SKB_CB(next_skb)->txstamp_ack;
3040 	}
3041 }
3042 
3043 /* Collapses two adjacent SKB's during retransmission. */
3044 static bool tcp_collapse_retrans(struct sock *sk, struct sk_buff *skb)
3045 {
3046 	struct tcp_sock *tp = tcp_sk(sk);
3047 	struct sk_buff *next_skb = skb_rb_next(skb);
3048 	int next_skb_size;
3049 
3050 	next_skb_size = next_skb->len;
3051 
3052 	BUG_ON(tcp_skb_pcount(skb) != 1 || tcp_skb_pcount(next_skb) != 1);
3053 
3054 	if (next_skb_size && !tcp_skb_shift(skb, next_skb, 1, next_skb_size))
3055 		return false;
3056 
3057 	tcp_highest_sack_replace(sk, next_skb, skb);
3058 
3059 	/* Update sequence range on original skb. */
3060 	TCP_SKB_CB(skb)->end_seq = TCP_SKB_CB(next_skb)->end_seq;
3061 
3062 	/* Merge over control information. This moves PSH/FIN etc. over */
3063 	TCP_SKB_CB(skb)->tcp_flags |= TCP_SKB_CB(next_skb)->tcp_flags;
3064 
3065 	/* All done, get rid of second SKB and account for it so
3066 	 * packet counting does not break.
3067 	 */
3068 	TCP_SKB_CB(skb)->sacked |= TCP_SKB_CB(next_skb)->sacked & TCPCB_EVER_RETRANS;
3069 	TCP_SKB_CB(skb)->eor = TCP_SKB_CB(next_skb)->eor;
3070 
3071 	/* changed transmit queue under us so clear hints */
3072 	tcp_clear_retrans_hints_partial(tp);
3073 	if (next_skb == tp->retransmit_skb_hint)
3074 		tp->retransmit_skb_hint = skb;
3075 
3076 	tcp_adjust_pcount(sk, next_skb, tcp_skb_pcount(next_skb));
3077 
3078 	tcp_skb_collapse_tstamp(skb, next_skb);
3079 
3080 	tcp_rtx_queue_unlink_and_free(next_skb, sk);
3081 	return true;
3082 }
3083 
3084 /* Check if coalescing SKBs is legal. */
3085 static bool tcp_can_collapse(const struct sock *sk, const struct sk_buff *skb)
3086 {
3087 	if (tcp_skb_pcount(skb) > 1)
3088 		return false;
3089 	if (skb_cloned(skb))
3090 		return false;
3091 	/* Some heuristics for collapsing over SACK'd could be invented */
3092 	if (TCP_SKB_CB(skb)->sacked & TCPCB_SACKED_ACKED)
3093 		return false;
3094 
3095 	return true;
3096 }
3097 
3098 /* Collapse packets in the retransmit queue to make to create
3099  * less packets on the wire. This is only done on retransmission.
3100  */
3101 static void tcp_retrans_try_collapse(struct sock *sk, struct sk_buff *to,
3102 				     int space)
3103 {
3104 	struct tcp_sock *tp = tcp_sk(sk);
3105 	struct sk_buff *skb = to, *tmp;
3106 	bool first = true;
3107 
3108 	if (!READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_retrans_collapse))
3109 		return;
3110 	if (TCP_SKB_CB(skb)->tcp_flags & TCPHDR_SYN)
3111 		return;
3112 
3113 	skb_rbtree_walk_from_safe(skb, tmp) {
3114 		if (!tcp_can_collapse(sk, skb))
3115 			break;
3116 
3117 		if (!tcp_skb_can_collapse(to, skb))
3118 			break;
3119 
3120 		space -= skb->len;
3121 
3122 		if (first) {
3123 			first = false;
3124 			continue;
3125 		}
3126 
3127 		if (space < 0)
3128 			break;
3129 
3130 		if (after(TCP_SKB_CB(skb)->end_seq, tcp_wnd_end(tp)))
3131 			break;
3132 
3133 		if (!tcp_collapse_retrans(sk, to))
3134 			break;
3135 	}
3136 }
3137 
3138 /* This retransmits one SKB.  Policy decisions and retransmit queue
3139  * state updates are done by the caller.  Returns non-zero if an
3140  * error occurred which prevented the send.
3141  */
3142 int __tcp_retransmit_skb(struct sock *sk, struct sk_buff *skb, int segs)
3143 {
3144 	struct inet_connection_sock *icsk = inet_csk(sk);
3145 	struct tcp_sock *tp = tcp_sk(sk);
3146 	unsigned int cur_mss;
3147 	int diff, len, err;
3148 
3149 
3150 	/* Inconclusive MTU probe */
3151 	if (icsk->icsk_mtup.probe_size)
3152 		icsk->icsk_mtup.probe_size = 0;
3153 
3154 	if (skb_still_in_host_queue(sk, skb))
3155 		return -EBUSY;
3156 
3157 	if (before(TCP_SKB_CB(skb)->seq, tp->snd_una)) {
3158 		if (unlikely(before(TCP_SKB_CB(skb)->end_seq, tp->snd_una))) {
3159 			WARN_ON_ONCE(1);
3160 			return -EINVAL;
3161 		}
3162 		if (tcp_trim_head(sk, skb, tp->snd_una - TCP_SKB_CB(skb)->seq))
3163 			return -ENOMEM;
3164 	}
3165 
3166 	if (inet_csk(sk)->icsk_af_ops->rebuild_header(sk))
3167 		return -EHOSTUNREACH; /* Routing failure or similar. */
3168 
3169 	cur_mss = tcp_current_mss(sk);
3170 
3171 	/* If receiver has shrunk his window, and skb is out of
3172 	 * new window, do not retransmit it. The exception is the
3173 	 * case, when window is shrunk to zero. In this case
3174 	 * our retransmit serves as a zero window probe.
3175 	 */
3176 	if (!before(TCP_SKB_CB(skb)->seq, tcp_wnd_end(tp)) &&
3177 	    TCP_SKB_CB(skb)->seq != tp->snd_una)
3178 		return -EAGAIN;
3179 
3180 	len = cur_mss * segs;
3181 	if (skb->len > len) {
3182 		if (tcp_fragment(sk, TCP_FRAG_IN_RTX_QUEUE, skb, len,
3183 				 cur_mss, GFP_ATOMIC))
3184 			return -ENOMEM; /* We'll try again later. */
3185 	} else {
3186 		if (skb_unclone_keeptruesize(skb, GFP_ATOMIC))
3187 			return -ENOMEM;
3188 
3189 		diff = tcp_skb_pcount(skb);
3190 		tcp_set_skb_tso_segs(skb, cur_mss);
3191 		diff -= tcp_skb_pcount(skb);
3192 		if (diff)
3193 			tcp_adjust_pcount(sk, skb, diff);
3194 		if (skb->len < cur_mss)
3195 			tcp_retrans_try_collapse(sk, skb, cur_mss);
3196 	}
3197 
3198 	/* RFC3168, section 6.1.1.1. ECN fallback */
3199 	if ((TCP_SKB_CB(skb)->tcp_flags & TCPHDR_SYN_ECN) == TCPHDR_SYN_ECN)
3200 		tcp_ecn_clear_syn(sk, skb);
3201 
3202 	/* Update global and local TCP statistics. */
3203 	segs = tcp_skb_pcount(skb);
3204 	TCP_ADD_STATS(sock_net(sk), TCP_MIB_RETRANSSEGS, segs);
3205 	if (TCP_SKB_CB(skb)->tcp_flags & TCPHDR_SYN)
3206 		__NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPSYNRETRANS);
3207 	tp->total_retrans += segs;
3208 	tp->bytes_retrans += skb->len;
3209 
3210 	/* make sure skb->data is aligned on arches that require it
3211 	 * and check if ack-trimming & collapsing extended the headroom
3212 	 * beyond what csum_start can cover.
3213 	 */
3214 	if (unlikely((NET_IP_ALIGN && ((unsigned long)skb->data & 3)) ||
3215 		     skb_headroom(skb) >= 0xFFFF)) {
3216 		struct sk_buff *nskb;
3217 
3218 		tcp_skb_tsorted_save(skb) {
3219 			nskb = __pskb_copy(skb, MAX_TCP_HEADER, GFP_ATOMIC);
3220 			if (nskb) {
3221 				nskb->dev = NULL;
3222 				err = tcp_transmit_skb(sk, nskb, 0, GFP_ATOMIC);
3223 			} else {
3224 				err = -ENOBUFS;
3225 			}
3226 		} tcp_skb_tsorted_restore(skb);
3227 
3228 		if (!err) {
3229 			tcp_update_skb_after_send(sk, skb, tp->tcp_wstamp_ns);
3230 			tcp_rate_skb_sent(sk, skb);
3231 		}
3232 	} else {
3233 		err = tcp_transmit_skb(sk, skb, 1, GFP_ATOMIC);
3234 	}
3235 
3236 	/* To avoid taking spuriously low RTT samples based on a timestamp
3237 	 * for a transmit that never happened, always mark EVER_RETRANS
3238 	 */
3239 	TCP_SKB_CB(skb)->sacked |= TCPCB_EVER_RETRANS;
3240 
3241 	if (BPF_SOCK_OPS_TEST_FLAG(tp, BPF_SOCK_OPS_RETRANS_CB_FLAG))
3242 		tcp_call_bpf_3arg(sk, BPF_SOCK_OPS_RETRANS_CB,
3243 				  TCP_SKB_CB(skb)->seq, segs, err);
3244 
3245 	if (likely(!err)) {
3246 		trace_tcp_retransmit_skb(sk, skb);
3247 	} else if (err != -EBUSY) {
3248 		NET_ADD_STATS(sock_net(sk), LINUX_MIB_TCPRETRANSFAIL, segs);
3249 	}
3250 	return err;
3251 }
3252 
3253 int tcp_retransmit_skb(struct sock *sk, struct sk_buff *skb, int segs)
3254 {
3255 	struct tcp_sock *tp = tcp_sk(sk);
3256 	int err = __tcp_retransmit_skb(sk, skb, segs);
3257 
3258 	if (err == 0) {
3259 #if FASTRETRANS_DEBUG > 0
3260 		if (TCP_SKB_CB(skb)->sacked & TCPCB_SACKED_RETRANS) {
3261 			net_dbg_ratelimited("retrans_out leaked\n");
3262 		}
3263 #endif
3264 		TCP_SKB_CB(skb)->sacked |= TCPCB_RETRANS;
3265 		tp->retrans_out += tcp_skb_pcount(skb);
3266 	}
3267 
3268 	/* Save stamp of the first (attempted) retransmit. */
3269 	if (!tp->retrans_stamp)
3270 		tp->retrans_stamp = tcp_skb_timestamp(skb);
3271 
3272 	if (tp->undo_retrans < 0)
3273 		tp->undo_retrans = 0;
3274 	tp->undo_retrans += tcp_skb_pcount(skb);
3275 	return err;
3276 }
3277 
3278 /* This gets called after a retransmit timeout, and the initially
3279  * retransmitted data is acknowledged.  It tries to continue
3280  * resending the rest of the retransmit queue, until either
3281  * we've sent it all or the congestion window limit is reached.
3282  */
3283 void tcp_xmit_retransmit_queue(struct sock *sk)
3284 {
3285 	const struct inet_connection_sock *icsk = inet_csk(sk);
3286 	struct sk_buff *skb, *rtx_head, *hole = NULL;
3287 	struct tcp_sock *tp = tcp_sk(sk);
3288 	bool rearm_timer = false;
3289 	u32 max_segs;
3290 	int mib_idx;
3291 
3292 	if (!tp->packets_out)
3293 		return;
3294 
3295 	rtx_head = tcp_rtx_queue_head(sk);
3296 	skb = tp->retransmit_skb_hint ?: rtx_head;
3297 	max_segs = tcp_tso_segs(sk, tcp_current_mss(sk));
3298 	skb_rbtree_walk_from(skb) {
3299 		__u8 sacked;
3300 		int segs;
3301 
3302 		if (tcp_pacing_check(sk))
3303 			break;
3304 
3305 		/* we could do better than to assign each time */
3306 		if (!hole)
3307 			tp->retransmit_skb_hint = skb;
3308 
3309 		segs = tcp_snd_cwnd(tp) - tcp_packets_in_flight(tp);
3310 		if (segs <= 0)
3311 			break;
3312 		sacked = TCP_SKB_CB(skb)->sacked;
3313 		/* In case tcp_shift_skb_data() have aggregated large skbs,
3314 		 * we need to make sure not sending too bigs TSO packets
3315 		 */
3316 		segs = min_t(int, segs, max_segs);
3317 
3318 		if (tp->retrans_out >= tp->lost_out) {
3319 			break;
3320 		} else if (!(sacked & TCPCB_LOST)) {
3321 			if (!hole && !(sacked & (TCPCB_SACKED_RETRANS|TCPCB_SACKED_ACKED)))
3322 				hole = skb;
3323 			continue;
3324 
3325 		} else {
3326 			if (icsk->icsk_ca_state != TCP_CA_Loss)
3327 				mib_idx = LINUX_MIB_TCPFASTRETRANS;
3328 			else
3329 				mib_idx = LINUX_MIB_TCPSLOWSTARTRETRANS;
3330 		}
3331 
3332 		if (sacked & (TCPCB_SACKED_ACKED|TCPCB_SACKED_RETRANS))
3333 			continue;
3334 
3335 		if (tcp_small_queue_check(sk, skb, 1))
3336 			break;
3337 
3338 		if (tcp_retransmit_skb(sk, skb, segs))
3339 			break;
3340 
3341 		NET_ADD_STATS(sock_net(sk), mib_idx, tcp_skb_pcount(skb));
3342 
3343 		if (tcp_in_cwnd_reduction(sk))
3344 			tp->prr_out += tcp_skb_pcount(skb);
3345 
3346 		if (skb == rtx_head &&
3347 		    icsk->icsk_pending != ICSK_TIME_REO_TIMEOUT)
3348 			rearm_timer = true;
3349 
3350 	}
3351 	if (rearm_timer)
3352 		tcp_reset_xmit_timer(sk, ICSK_TIME_RETRANS,
3353 				     inet_csk(sk)->icsk_rto,
3354 				     TCP_RTO_MAX);
3355 }
3356 
3357 /* We allow to exceed memory limits for FIN packets to expedite
3358  * connection tear down and (memory) recovery.
3359  * Otherwise tcp_send_fin() could be tempted to either delay FIN
3360  * or even be forced to close flow without any FIN.
3361  * In general, we want to allow one skb per socket to avoid hangs
3362  * with edge trigger epoll()
3363  */
3364 void sk_forced_mem_schedule(struct sock *sk, int size)
3365 {
3366 	int amt;
3367 
3368 	if (size <= sk->sk_forward_alloc)
3369 		return;
3370 	amt = sk_mem_pages(size);
3371 	sk->sk_forward_alloc += amt * SK_MEM_QUANTUM;
3372 	sk_memory_allocated_add(sk, amt);
3373 
3374 	if (mem_cgroup_sockets_enabled && sk->sk_memcg)
3375 		mem_cgroup_charge_skmem(sk->sk_memcg, amt,
3376 					gfp_memcg_charge() | __GFP_NOFAIL);
3377 }
3378 
3379 /* Send a FIN. The caller locks the socket for us.
3380  * We should try to send a FIN packet really hard, but eventually give up.
3381  */
3382 void tcp_send_fin(struct sock *sk)
3383 {
3384 	struct sk_buff *skb, *tskb, *tail = tcp_write_queue_tail(sk);
3385 	struct tcp_sock *tp = tcp_sk(sk);
3386 
3387 	/* Optimization, tack on the FIN if we have one skb in write queue and
3388 	 * this skb was not yet sent, or we are under memory pressure.
3389 	 * Note: in the latter case, FIN packet will be sent after a timeout,
3390 	 * as TCP stack thinks it has already been transmitted.
3391 	 */
3392 	tskb = tail;
3393 	if (!tskb && tcp_under_memory_pressure(sk))
3394 		tskb = skb_rb_last(&sk->tcp_rtx_queue);
3395 
3396 	if (tskb) {
3397 		TCP_SKB_CB(tskb)->tcp_flags |= TCPHDR_FIN;
3398 		TCP_SKB_CB(tskb)->end_seq++;
3399 		tp->write_seq++;
3400 		if (!tail) {
3401 			/* This means tskb was already sent.
3402 			 * Pretend we included the FIN on previous transmit.
3403 			 * We need to set tp->snd_nxt to the value it would have
3404 			 * if FIN had been sent. This is because retransmit path
3405 			 * does not change tp->snd_nxt.
3406 			 */
3407 			WRITE_ONCE(tp->snd_nxt, tp->snd_nxt + 1);
3408 			return;
3409 		}
3410 	} else {
3411 		skb = alloc_skb_fclone(MAX_TCP_HEADER, sk->sk_allocation);
3412 		if (unlikely(!skb))
3413 			return;
3414 
3415 		INIT_LIST_HEAD(&skb->tcp_tsorted_anchor);
3416 		skb_reserve(skb, MAX_TCP_HEADER);
3417 		sk_forced_mem_schedule(sk, skb->truesize);
3418 		/* FIN eats a sequence byte, write_seq advanced by tcp_queue_skb(). */
3419 		tcp_init_nondata_skb(skb, tp->write_seq,
3420 				     TCPHDR_ACK | TCPHDR_FIN);
3421 		tcp_queue_skb(sk, skb);
3422 	}
3423 	__tcp_push_pending_frames(sk, tcp_current_mss(sk), TCP_NAGLE_OFF);
3424 }
3425 
3426 /* We get here when a process closes a file descriptor (either due to
3427  * an explicit close() or as a byproduct of exit()'ing) and there
3428  * was unread data in the receive queue.  This behavior is recommended
3429  * by RFC 2525, section 2.17.  -DaveM
3430  */
3431 void tcp_send_active_reset(struct sock *sk, gfp_t priority)
3432 {
3433 	struct sk_buff *skb;
3434 
3435 	TCP_INC_STATS(sock_net(sk), TCP_MIB_OUTRSTS);
3436 
3437 	/* NOTE: No TCP options attached and we never retransmit this. */
3438 	skb = alloc_skb(MAX_TCP_HEADER, priority);
3439 	if (!skb) {
3440 		NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPABORTFAILED);
3441 		return;
3442 	}
3443 
3444 	/* Reserve space for headers and prepare control bits. */
3445 	skb_reserve(skb, MAX_TCP_HEADER);
3446 	tcp_init_nondata_skb(skb, tcp_acceptable_seq(sk),
3447 			     TCPHDR_ACK | TCPHDR_RST);
3448 	tcp_mstamp_refresh(tcp_sk(sk));
3449 	/* Send it off. */
3450 	if (tcp_transmit_skb(sk, skb, 0, priority))
3451 		NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPABORTFAILED);
3452 
3453 	/* skb of trace_tcp_send_reset() keeps the skb that caused RST,
3454 	 * skb here is different to the troublesome skb, so use NULL
3455 	 */
3456 	trace_tcp_send_reset(sk, NULL);
3457 }
3458 
3459 /* Send a crossed SYN-ACK during socket establishment.
3460  * WARNING: This routine must only be called when we have already sent
3461  * a SYN packet that crossed the incoming SYN that caused this routine
3462  * to get called. If this assumption fails then the initial rcv_wnd
3463  * and rcv_wscale values will not be correct.
3464  */
3465 int tcp_send_synack(struct sock *sk)
3466 {
3467 	struct sk_buff *skb;
3468 
3469 	skb = tcp_rtx_queue_head(sk);
3470 	if (!skb || !(TCP_SKB_CB(skb)->tcp_flags & TCPHDR_SYN)) {
3471 		pr_err("%s: wrong queue state\n", __func__);
3472 		return -EFAULT;
3473 	}
3474 	if (!(TCP_SKB_CB(skb)->tcp_flags & TCPHDR_ACK)) {
3475 		if (skb_cloned(skb)) {
3476 			struct sk_buff *nskb;
3477 
3478 			tcp_skb_tsorted_save(skb) {
3479 				nskb = skb_copy(skb, GFP_ATOMIC);
3480 			} tcp_skb_tsorted_restore(skb);
3481 			if (!nskb)
3482 				return -ENOMEM;
3483 			INIT_LIST_HEAD(&nskb->tcp_tsorted_anchor);
3484 			tcp_highest_sack_replace(sk, skb, nskb);
3485 			tcp_rtx_queue_unlink_and_free(skb, sk);
3486 			__skb_header_release(nskb);
3487 			tcp_rbtree_insert(&sk->tcp_rtx_queue, nskb);
3488 			sk_wmem_queued_add(sk, nskb->truesize);
3489 			sk_mem_charge(sk, nskb->truesize);
3490 			skb = nskb;
3491 		}
3492 
3493 		TCP_SKB_CB(skb)->tcp_flags |= TCPHDR_ACK;
3494 		tcp_ecn_send_synack(sk, skb);
3495 	}
3496 	return tcp_transmit_skb(sk, skb, 1, GFP_ATOMIC);
3497 }
3498 
3499 /**
3500  * tcp_make_synack - Allocate one skb and build a SYNACK packet.
3501  * @sk: listener socket
3502  * @dst: dst entry attached to the SYNACK. It is consumed and caller
3503  *       should not use it again.
3504  * @req: request_sock pointer
3505  * @foc: cookie for tcp fast open
3506  * @synack_type: Type of synack to prepare
3507  * @syn_skb: SYN packet just received.  It could be NULL for rtx case.
3508  */
3509 struct sk_buff *tcp_make_synack(const struct sock *sk, struct dst_entry *dst,
3510 				struct request_sock *req,
3511 				struct tcp_fastopen_cookie *foc,
3512 				enum tcp_synack_type synack_type,
3513 				struct sk_buff *syn_skb)
3514 {
3515 	struct inet_request_sock *ireq = inet_rsk(req);
3516 	const struct tcp_sock *tp = tcp_sk(sk);
3517 	struct tcp_md5sig_key *md5 = NULL;
3518 	struct tcp_out_options opts;
3519 	struct sk_buff *skb;
3520 	int tcp_header_size;
3521 	struct tcphdr *th;
3522 	int mss;
3523 	u64 now;
3524 
3525 	skb = alloc_skb(MAX_TCP_HEADER, GFP_ATOMIC);
3526 	if (unlikely(!skb)) {
3527 		dst_release(dst);
3528 		return NULL;
3529 	}
3530 	/* Reserve space for headers. */
3531 	skb_reserve(skb, MAX_TCP_HEADER);
3532 
3533 	switch (synack_type) {
3534 	case TCP_SYNACK_NORMAL:
3535 		skb_set_owner_w(skb, req_to_sk(req));
3536 		break;
3537 	case TCP_SYNACK_COOKIE:
3538 		/* Under synflood, we do not attach skb to a socket,
3539 		 * to avoid false sharing.
3540 		 */
3541 		break;
3542 	case TCP_SYNACK_FASTOPEN:
3543 		/* sk is a const pointer, because we want to express multiple
3544 		 * cpu might call us concurrently.
3545 		 * sk->sk_wmem_alloc in an atomic, we can promote to rw.
3546 		 */
3547 		skb_set_owner_w(skb, (struct sock *)sk);
3548 		break;
3549 	}
3550 	skb_dst_set(skb, dst);
3551 
3552 	mss = tcp_mss_clamp(tp, dst_metric_advmss(dst));
3553 
3554 	memset(&opts, 0, sizeof(opts));
3555 	now = tcp_clock_ns();
3556 #ifdef CONFIG_SYN_COOKIES
3557 	if (unlikely(synack_type == TCP_SYNACK_COOKIE && ireq->tstamp_ok))
3558 		skb_set_delivery_time(skb, cookie_init_timestamp(req, now),
3559 				      true);
3560 	else
3561 #endif
3562 	{
3563 		skb_set_delivery_time(skb, now, true);
3564 		if (!tcp_rsk(req)->snt_synack) /* Timestamp first SYNACK */
3565 			tcp_rsk(req)->snt_synack = tcp_skb_timestamp_us(skb);
3566 	}
3567 
3568 #ifdef CONFIG_TCP_MD5SIG
3569 	rcu_read_lock();
3570 	md5 = tcp_rsk(req)->af_specific->req_md5_lookup(sk, req_to_sk(req));
3571 #endif
3572 	skb_set_hash(skb, tcp_rsk(req)->txhash, PKT_HASH_TYPE_L4);
3573 	/* bpf program will be interested in the tcp_flags */
3574 	TCP_SKB_CB(skb)->tcp_flags = TCPHDR_SYN | TCPHDR_ACK;
3575 	tcp_header_size = tcp_synack_options(sk, req, mss, skb, &opts, md5,
3576 					     foc, synack_type,
3577 					     syn_skb) + sizeof(*th);
3578 
3579 	skb_push(skb, tcp_header_size);
3580 	skb_reset_transport_header(skb);
3581 
3582 	th = (struct tcphdr *)skb->data;
3583 	memset(th, 0, sizeof(struct tcphdr));
3584 	th->syn = 1;
3585 	th->ack = 1;
3586 	tcp_ecn_make_synack(req, th);
3587 	th->source = htons(ireq->ir_num);
3588 	th->dest = ireq->ir_rmt_port;
3589 	skb->mark = ireq->ir_mark;
3590 	skb->ip_summed = CHECKSUM_PARTIAL;
3591 	th->seq = htonl(tcp_rsk(req)->snt_isn);
3592 	/* XXX data is queued and acked as is. No buffer/window check */
3593 	th->ack_seq = htonl(tcp_rsk(req)->rcv_nxt);
3594 
3595 	/* RFC1323: The window in SYN & SYN/ACK segments is never scaled. */
3596 	th->window = htons(min(req->rsk_rcv_wnd, 65535U));
3597 	tcp_options_write(th, NULL, &opts);
3598 	th->doff = (tcp_header_size >> 2);
3599 	__TCP_INC_STATS(sock_net(sk), TCP_MIB_OUTSEGS);
3600 
3601 #ifdef CONFIG_TCP_MD5SIG
3602 	/* Okay, we have all we need - do the md5 hash if needed */
3603 	if (md5)
3604 		tcp_rsk(req)->af_specific->calc_md5_hash(opts.hash_location,
3605 					       md5, req_to_sk(req), skb);
3606 	rcu_read_unlock();
3607 #endif
3608 
3609 	bpf_skops_write_hdr_opt((struct sock *)sk, skb, req, syn_skb,
3610 				synack_type, &opts);
3611 
3612 	skb_set_delivery_time(skb, now, true);
3613 	tcp_add_tx_delay(skb, tp);
3614 
3615 	return skb;
3616 }
3617 EXPORT_SYMBOL(tcp_make_synack);
3618 
3619 static void tcp_ca_dst_init(struct sock *sk, const struct dst_entry *dst)
3620 {
3621 	struct inet_connection_sock *icsk = inet_csk(sk);
3622 	const struct tcp_congestion_ops *ca;
3623 	u32 ca_key = dst_metric(dst, RTAX_CC_ALGO);
3624 
3625 	if (ca_key == TCP_CA_UNSPEC)
3626 		return;
3627 
3628 	rcu_read_lock();
3629 	ca = tcp_ca_find_key(ca_key);
3630 	if (likely(ca && bpf_try_module_get(ca, ca->owner))) {
3631 		bpf_module_put(icsk->icsk_ca_ops, icsk->icsk_ca_ops->owner);
3632 		icsk->icsk_ca_dst_locked = tcp_ca_dst_locked(dst);
3633 		icsk->icsk_ca_ops = ca;
3634 	}
3635 	rcu_read_unlock();
3636 }
3637 
3638 /* Do all connect socket setups that can be done AF independent. */
3639 static void tcp_connect_init(struct sock *sk)
3640 {
3641 	const struct dst_entry *dst = __sk_dst_get(sk);
3642 	struct tcp_sock *tp = tcp_sk(sk);
3643 	__u8 rcv_wscale;
3644 	u32 rcv_wnd;
3645 
3646 	/* We'll fix this up when we get a response from the other end.
3647 	 * See tcp_input.c:tcp_rcv_state_process case TCP_SYN_SENT.
3648 	 */
3649 	tp->tcp_header_len = sizeof(struct tcphdr);
3650 	if (READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_timestamps))
3651 		tp->tcp_header_len += TCPOLEN_TSTAMP_ALIGNED;
3652 
3653 #ifdef CONFIG_TCP_MD5SIG
3654 	if (tp->af_specific->md5_lookup(sk, sk))
3655 		tp->tcp_header_len += TCPOLEN_MD5SIG_ALIGNED;
3656 #endif
3657 
3658 	/* If user gave his TCP_MAXSEG, record it to clamp */
3659 	if (tp->rx_opt.user_mss)
3660 		tp->rx_opt.mss_clamp = tp->rx_opt.user_mss;
3661 	tp->max_window = 0;
3662 	tcp_mtup_init(sk);
3663 	tcp_sync_mss(sk, dst_mtu(dst));
3664 
3665 	tcp_ca_dst_init(sk, dst);
3666 
3667 	if (!tp->window_clamp)
3668 		tp->window_clamp = dst_metric(dst, RTAX_WINDOW);
3669 	tp->advmss = tcp_mss_clamp(tp, dst_metric_advmss(dst));
3670 
3671 	tcp_initialize_rcv_mss(sk);
3672 
3673 	/* limit the window selection if the user enforce a smaller rx buffer */
3674 	if (sk->sk_userlocks & SOCK_RCVBUF_LOCK &&
3675 	    (tp->window_clamp > tcp_full_space(sk) || tp->window_clamp == 0))
3676 		tp->window_clamp = tcp_full_space(sk);
3677 
3678 	rcv_wnd = tcp_rwnd_init_bpf(sk);
3679 	if (rcv_wnd == 0)
3680 		rcv_wnd = dst_metric(dst, RTAX_INITRWND);
3681 
3682 	tcp_select_initial_window(sk, tcp_full_space(sk),
3683 				  tp->advmss - (tp->rx_opt.ts_recent_stamp ? tp->tcp_header_len - sizeof(struct tcphdr) : 0),
3684 				  &tp->rcv_wnd,
3685 				  &tp->window_clamp,
3686 				  READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_window_scaling),
3687 				  &rcv_wscale,
3688 				  rcv_wnd);
3689 
3690 	tp->rx_opt.rcv_wscale = rcv_wscale;
3691 	tp->rcv_ssthresh = tp->rcv_wnd;
3692 
3693 	sk->sk_err = 0;
3694 	sock_reset_flag(sk, SOCK_DONE);
3695 	tp->snd_wnd = 0;
3696 	tcp_init_wl(tp, 0);
3697 	tcp_write_queue_purge(sk);
3698 	tp->snd_una = tp->write_seq;
3699 	tp->snd_sml = tp->write_seq;
3700 	tp->snd_up = tp->write_seq;
3701 	WRITE_ONCE(tp->snd_nxt, tp->write_seq);
3702 
3703 	if (likely(!tp->repair))
3704 		tp->rcv_nxt = 0;
3705 	else
3706 		tp->rcv_tstamp = tcp_jiffies32;
3707 	tp->rcv_wup = tp->rcv_nxt;
3708 	WRITE_ONCE(tp->copied_seq, tp->rcv_nxt);
3709 
3710 	inet_csk(sk)->icsk_rto = tcp_timeout_init(sk);
3711 	inet_csk(sk)->icsk_retransmits = 0;
3712 	tcp_clear_retrans(tp);
3713 }
3714 
3715 static void tcp_connect_queue_skb(struct sock *sk, struct sk_buff *skb)
3716 {
3717 	struct tcp_sock *tp = tcp_sk(sk);
3718 	struct tcp_skb_cb *tcb = TCP_SKB_CB(skb);
3719 
3720 	tcb->end_seq += skb->len;
3721 	__skb_header_release(skb);
3722 	sk_wmem_queued_add(sk, skb->truesize);
3723 	sk_mem_charge(sk, skb->truesize);
3724 	WRITE_ONCE(tp->write_seq, tcb->end_seq);
3725 	tp->packets_out += tcp_skb_pcount(skb);
3726 }
3727 
3728 /* Build and send a SYN with data and (cached) Fast Open cookie. However,
3729  * queue a data-only packet after the regular SYN, such that regular SYNs
3730  * are retransmitted on timeouts. Also if the remote SYN-ACK acknowledges
3731  * only the SYN sequence, the data are retransmitted in the first ACK.
3732  * If cookie is not cached or other error occurs, falls back to send a
3733  * regular SYN with Fast Open cookie request option.
3734  */
3735 static int tcp_send_syn_data(struct sock *sk, struct sk_buff *syn)
3736 {
3737 	struct inet_connection_sock *icsk = inet_csk(sk);
3738 	struct tcp_sock *tp = tcp_sk(sk);
3739 	struct tcp_fastopen_request *fo = tp->fastopen_req;
3740 	int space, err = 0;
3741 	struct sk_buff *syn_data;
3742 
3743 	tp->rx_opt.mss_clamp = tp->advmss;  /* If MSS is not cached */
3744 	if (!tcp_fastopen_cookie_check(sk, &tp->rx_opt.mss_clamp, &fo->cookie))
3745 		goto fallback;
3746 
3747 	/* MSS for SYN-data is based on cached MSS and bounded by PMTU and
3748 	 * user-MSS. Reserve maximum option space for middleboxes that add
3749 	 * private TCP options. The cost is reduced data space in SYN :(
3750 	 */
3751 	tp->rx_opt.mss_clamp = tcp_mss_clamp(tp, tp->rx_opt.mss_clamp);
3752 	/* Sync mss_cache after updating the mss_clamp */
3753 	tcp_sync_mss(sk, icsk->icsk_pmtu_cookie);
3754 
3755 	space = __tcp_mtu_to_mss(sk, icsk->icsk_pmtu_cookie) -
3756 		MAX_TCP_OPTION_SPACE;
3757 
3758 	space = min_t(size_t, space, fo->size);
3759 
3760 	/* limit to order-0 allocations */
3761 	space = min_t(size_t, space, SKB_MAX_HEAD(MAX_TCP_HEADER));
3762 
3763 	syn_data = tcp_stream_alloc_skb(sk, space, sk->sk_allocation, false);
3764 	if (!syn_data)
3765 		goto fallback;
3766 	memcpy(syn_data->cb, syn->cb, sizeof(syn->cb));
3767 	if (space) {
3768 		int copied = copy_from_iter(skb_put(syn_data, space), space,
3769 					    &fo->data->msg_iter);
3770 		if (unlikely(!copied)) {
3771 			tcp_skb_tsorted_anchor_cleanup(syn_data);
3772 			kfree_skb(syn_data);
3773 			goto fallback;
3774 		}
3775 		if (copied != space) {
3776 			skb_trim(syn_data, copied);
3777 			space = copied;
3778 		}
3779 		skb_zcopy_set(syn_data, fo->uarg, NULL);
3780 	}
3781 	/* No more data pending in inet_wait_for_connect() */
3782 	if (space == fo->size)
3783 		fo->data = NULL;
3784 	fo->copied = space;
3785 
3786 	tcp_connect_queue_skb(sk, syn_data);
3787 	if (syn_data->len)
3788 		tcp_chrono_start(sk, TCP_CHRONO_BUSY);
3789 
3790 	err = tcp_transmit_skb(sk, syn_data, 1, sk->sk_allocation);
3791 
3792 	skb_set_delivery_time(syn, syn_data->skb_mstamp_ns, true);
3793 
3794 	/* Now full SYN+DATA was cloned and sent (or not),
3795 	 * remove the SYN from the original skb (syn_data)
3796 	 * we keep in write queue in case of a retransmit, as we
3797 	 * also have the SYN packet (with no data) in the same queue.
3798 	 */
3799 	TCP_SKB_CB(syn_data)->seq++;
3800 	TCP_SKB_CB(syn_data)->tcp_flags = TCPHDR_ACK | TCPHDR_PSH;
3801 	if (!err) {
3802 		tp->syn_data = (fo->copied > 0);
3803 		tcp_rbtree_insert(&sk->tcp_rtx_queue, syn_data);
3804 		NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPORIGDATASENT);
3805 		goto done;
3806 	}
3807 
3808 	/* data was not sent, put it in write_queue */
3809 	__skb_queue_tail(&sk->sk_write_queue, syn_data);
3810 	tp->packets_out -= tcp_skb_pcount(syn_data);
3811 
3812 fallback:
3813 	/* Send a regular SYN with Fast Open cookie request option */
3814 	if (fo->cookie.len > 0)
3815 		fo->cookie.len = 0;
3816 	err = tcp_transmit_skb(sk, syn, 1, sk->sk_allocation);
3817 	if (err)
3818 		tp->syn_fastopen = 0;
3819 done:
3820 	fo->cookie.len = -1;  /* Exclude Fast Open option for SYN retries */
3821 	return err;
3822 }
3823 
3824 /* Build a SYN and send it off. */
3825 int tcp_connect(struct sock *sk)
3826 {
3827 	struct tcp_sock *tp = tcp_sk(sk);
3828 	struct sk_buff *buff;
3829 	int err;
3830 
3831 	tcp_call_bpf(sk, BPF_SOCK_OPS_TCP_CONNECT_CB, 0, NULL);
3832 
3833 	if (inet_csk(sk)->icsk_af_ops->rebuild_header(sk))
3834 		return -EHOSTUNREACH; /* Routing failure or similar. */
3835 
3836 	tcp_connect_init(sk);
3837 
3838 	if (unlikely(tp->repair)) {
3839 		tcp_finish_connect(sk, NULL);
3840 		return 0;
3841 	}
3842 
3843 	buff = tcp_stream_alloc_skb(sk, 0, sk->sk_allocation, true);
3844 	if (unlikely(!buff))
3845 		return -ENOBUFS;
3846 
3847 	tcp_init_nondata_skb(buff, tp->write_seq++, TCPHDR_SYN);
3848 	tcp_mstamp_refresh(tp);
3849 	tp->retrans_stamp = tcp_time_stamp(tp);
3850 	tcp_connect_queue_skb(sk, buff);
3851 	tcp_ecn_send_syn(sk, buff);
3852 	tcp_rbtree_insert(&sk->tcp_rtx_queue, buff);
3853 
3854 	/* Send off SYN; include data in Fast Open. */
3855 	err = tp->fastopen_req ? tcp_send_syn_data(sk, buff) :
3856 	      tcp_transmit_skb(sk, buff, 1, sk->sk_allocation);
3857 	if (err == -ECONNREFUSED)
3858 		return err;
3859 
3860 	/* We change tp->snd_nxt after the tcp_transmit_skb() call
3861 	 * in order to make this packet get counted in tcpOutSegs.
3862 	 */
3863 	WRITE_ONCE(tp->snd_nxt, tp->write_seq);
3864 	tp->pushed_seq = tp->write_seq;
3865 	buff = tcp_send_head(sk);
3866 	if (unlikely(buff)) {
3867 		WRITE_ONCE(tp->snd_nxt, TCP_SKB_CB(buff)->seq);
3868 		tp->pushed_seq	= TCP_SKB_CB(buff)->seq;
3869 	}
3870 	TCP_INC_STATS(sock_net(sk), TCP_MIB_ACTIVEOPENS);
3871 
3872 	/* Timer for repeating the SYN until an answer. */
3873 	inet_csk_reset_xmit_timer(sk, ICSK_TIME_RETRANS,
3874 				  inet_csk(sk)->icsk_rto, TCP_RTO_MAX);
3875 	return 0;
3876 }
3877 EXPORT_SYMBOL(tcp_connect);
3878 
3879 /* Send out a delayed ack, the caller does the policy checking
3880  * to see if we should even be here.  See tcp_input.c:tcp_ack_snd_check()
3881  * for details.
3882  */
3883 void tcp_send_delayed_ack(struct sock *sk)
3884 {
3885 	struct inet_connection_sock *icsk = inet_csk(sk);
3886 	int ato = icsk->icsk_ack.ato;
3887 	unsigned long timeout;
3888 
3889 	if (ato > TCP_DELACK_MIN) {
3890 		const struct tcp_sock *tp = tcp_sk(sk);
3891 		int max_ato = HZ / 2;
3892 
3893 		if (inet_csk_in_pingpong_mode(sk) ||
3894 		    (icsk->icsk_ack.pending & ICSK_ACK_PUSHED))
3895 			max_ato = TCP_DELACK_MAX;
3896 
3897 		/* Slow path, intersegment interval is "high". */
3898 
3899 		/* If some rtt estimate is known, use it to bound delayed ack.
3900 		 * Do not use inet_csk(sk)->icsk_rto here, use results of rtt measurements
3901 		 * directly.
3902 		 */
3903 		if (tp->srtt_us) {
3904 			int rtt = max_t(int, usecs_to_jiffies(tp->srtt_us >> 3),
3905 					TCP_DELACK_MIN);
3906 
3907 			if (rtt < max_ato)
3908 				max_ato = rtt;
3909 		}
3910 
3911 		ato = min(ato, max_ato);
3912 	}
3913 
3914 	ato = min_t(u32, ato, inet_csk(sk)->icsk_delack_max);
3915 
3916 	/* Stay within the limit we were given */
3917 	timeout = jiffies + ato;
3918 
3919 	/* Use new timeout only if there wasn't a older one earlier. */
3920 	if (icsk->icsk_ack.pending & ICSK_ACK_TIMER) {
3921 		/* If delack timer is about to expire, send ACK now. */
3922 		if (time_before_eq(icsk->icsk_ack.timeout, jiffies + (ato >> 2))) {
3923 			tcp_send_ack(sk);
3924 			return;
3925 		}
3926 
3927 		if (!time_before(timeout, icsk->icsk_ack.timeout))
3928 			timeout = icsk->icsk_ack.timeout;
3929 	}
3930 	icsk->icsk_ack.pending |= ICSK_ACK_SCHED | ICSK_ACK_TIMER;
3931 	icsk->icsk_ack.timeout = timeout;
3932 	sk_reset_timer(sk, &icsk->icsk_delack_timer, timeout);
3933 }
3934 
3935 /* This routine sends an ack and also updates the window. */
3936 void __tcp_send_ack(struct sock *sk, u32 rcv_nxt)
3937 {
3938 	struct sk_buff *buff;
3939 
3940 	/* If we have been reset, we may not send again. */
3941 	if (sk->sk_state == TCP_CLOSE)
3942 		return;
3943 
3944 	/* We are not putting this on the write queue, so
3945 	 * tcp_transmit_skb() will set the ownership to this
3946 	 * sock.
3947 	 */
3948 	buff = alloc_skb(MAX_TCP_HEADER,
3949 			 sk_gfp_mask(sk, GFP_ATOMIC | __GFP_NOWARN));
3950 	if (unlikely(!buff)) {
3951 		struct inet_connection_sock *icsk = inet_csk(sk);
3952 		unsigned long delay;
3953 
3954 		delay = TCP_DELACK_MAX << icsk->icsk_ack.retry;
3955 		if (delay < TCP_RTO_MAX)
3956 			icsk->icsk_ack.retry++;
3957 		inet_csk_schedule_ack(sk);
3958 		icsk->icsk_ack.ato = TCP_ATO_MIN;
3959 		inet_csk_reset_xmit_timer(sk, ICSK_TIME_DACK, delay, TCP_RTO_MAX);
3960 		return;
3961 	}
3962 
3963 	/* Reserve space for headers and prepare control bits. */
3964 	skb_reserve(buff, MAX_TCP_HEADER);
3965 	tcp_init_nondata_skb(buff, tcp_acceptable_seq(sk), TCPHDR_ACK);
3966 
3967 	/* We do not want pure acks influencing TCP Small Queues or fq/pacing
3968 	 * too much.
3969 	 * SKB_TRUESIZE(max(1 .. 66, MAX_TCP_HEADER)) is unfortunately ~784
3970 	 */
3971 	skb_set_tcp_pure_ack(buff);
3972 
3973 	/* Send it off, this clears delayed acks for us. */
3974 	__tcp_transmit_skb(sk, buff, 0, (__force gfp_t)0, rcv_nxt);
3975 }
3976 EXPORT_SYMBOL_GPL(__tcp_send_ack);
3977 
3978 void tcp_send_ack(struct sock *sk)
3979 {
3980 	__tcp_send_ack(sk, tcp_sk(sk)->rcv_nxt);
3981 }
3982 
3983 /* This routine sends a packet with an out of date sequence
3984  * number. It assumes the other end will try to ack it.
3985  *
3986  * Question: what should we make while urgent mode?
3987  * 4.4BSD forces sending single byte of data. We cannot send
3988  * out of window data, because we have SND.NXT==SND.MAX...
3989  *
3990  * Current solution: to send TWO zero-length segments in urgent mode:
3991  * one is with SEG.SEQ=SND.UNA to deliver urgent pointer, another is
3992  * out-of-date with SND.UNA-1 to probe window.
3993  */
3994 static int tcp_xmit_probe_skb(struct sock *sk, int urgent, int mib)
3995 {
3996 	struct tcp_sock *tp = tcp_sk(sk);
3997 	struct sk_buff *skb;
3998 
3999 	/* We don't queue it, tcp_transmit_skb() sets ownership. */
4000 	skb = alloc_skb(MAX_TCP_HEADER,
4001 			sk_gfp_mask(sk, GFP_ATOMIC | __GFP_NOWARN));
4002 	if (!skb)
4003 		return -1;
4004 
4005 	/* Reserve space for headers and set control bits. */
4006 	skb_reserve(skb, MAX_TCP_HEADER);
4007 	/* Use a previous sequence.  This should cause the other
4008 	 * end to send an ack.  Don't queue or clone SKB, just
4009 	 * send it.
4010 	 */
4011 	tcp_init_nondata_skb(skb, tp->snd_una - !urgent, TCPHDR_ACK);
4012 	NET_INC_STATS(sock_net(sk), mib);
4013 	return tcp_transmit_skb(sk, skb, 0, (__force gfp_t)0);
4014 }
4015 
4016 /* Called from setsockopt( ... TCP_REPAIR ) */
4017 void tcp_send_window_probe(struct sock *sk)
4018 {
4019 	if (sk->sk_state == TCP_ESTABLISHED) {
4020 		tcp_sk(sk)->snd_wl1 = tcp_sk(sk)->rcv_nxt - 1;
4021 		tcp_mstamp_refresh(tcp_sk(sk));
4022 		tcp_xmit_probe_skb(sk, 0, LINUX_MIB_TCPWINPROBE);
4023 	}
4024 }
4025 
4026 /* Initiate keepalive or window probe from timer. */
4027 int tcp_write_wakeup(struct sock *sk, int mib)
4028 {
4029 	struct tcp_sock *tp = tcp_sk(sk);
4030 	struct sk_buff *skb;
4031 
4032 	if (sk->sk_state == TCP_CLOSE)
4033 		return -1;
4034 
4035 	skb = tcp_send_head(sk);
4036 	if (skb && before(TCP_SKB_CB(skb)->seq, tcp_wnd_end(tp))) {
4037 		int err;
4038 		unsigned int mss = tcp_current_mss(sk);
4039 		unsigned int seg_size = tcp_wnd_end(tp) - TCP_SKB_CB(skb)->seq;
4040 
4041 		if (before(tp->pushed_seq, TCP_SKB_CB(skb)->end_seq))
4042 			tp->pushed_seq = TCP_SKB_CB(skb)->end_seq;
4043 
4044 		/* We are probing the opening of a window
4045 		 * but the window size is != 0
4046 		 * must have been a result SWS avoidance ( sender )
4047 		 */
4048 		if (seg_size < TCP_SKB_CB(skb)->end_seq - TCP_SKB_CB(skb)->seq ||
4049 		    skb->len > mss) {
4050 			seg_size = min(seg_size, mss);
4051 			TCP_SKB_CB(skb)->tcp_flags |= TCPHDR_PSH;
4052 			if (tcp_fragment(sk, TCP_FRAG_IN_WRITE_QUEUE,
4053 					 skb, seg_size, mss, GFP_ATOMIC))
4054 				return -1;
4055 		} else if (!tcp_skb_pcount(skb))
4056 			tcp_set_skb_tso_segs(skb, mss);
4057 
4058 		TCP_SKB_CB(skb)->tcp_flags |= TCPHDR_PSH;
4059 		err = tcp_transmit_skb(sk, skb, 1, GFP_ATOMIC);
4060 		if (!err)
4061 			tcp_event_new_data_sent(sk, skb);
4062 		return err;
4063 	} else {
4064 		if (between(tp->snd_up, tp->snd_una + 1, tp->snd_una + 0xFFFF))
4065 			tcp_xmit_probe_skb(sk, 1, mib);
4066 		return tcp_xmit_probe_skb(sk, 0, mib);
4067 	}
4068 }
4069 
4070 /* A window probe timeout has occurred.  If window is not closed send
4071  * a partial packet else a zero probe.
4072  */
4073 void tcp_send_probe0(struct sock *sk)
4074 {
4075 	struct inet_connection_sock *icsk = inet_csk(sk);
4076 	struct tcp_sock *tp = tcp_sk(sk);
4077 	struct net *net = sock_net(sk);
4078 	unsigned long timeout;
4079 	int err;
4080 
4081 	err = tcp_write_wakeup(sk, LINUX_MIB_TCPWINPROBE);
4082 
4083 	if (tp->packets_out || tcp_write_queue_empty(sk)) {
4084 		/* Cancel probe timer, if it is not required. */
4085 		icsk->icsk_probes_out = 0;
4086 		icsk->icsk_backoff = 0;
4087 		icsk->icsk_probes_tstamp = 0;
4088 		return;
4089 	}
4090 
4091 	icsk->icsk_probes_out++;
4092 	if (err <= 0) {
4093 		if (icsk->icsk_backoff < READ_ONCE(net->ipv4.sysctl_tcp_retries2))
4094 			icsk->icsk_backoff++;
4095 		timeout = tcp_probe0_when(sk, TCP_RTO_MAX);
4096 	} else {
4097 		/* If packet was not sent due to local congestion,
4098 		 * Let senders fight for local resources conservatively.
4099 		 */
4100 		timeout = TCP_RESOURCE_PROBE_INTERVAL;
4101 	}
4102 
4103 	timeout = tcp_clamp_probe0_to_user_timeout(sk, timeout);
4104 	tcp_reset_xmit_timer(sk, ICSK_TIME_PROBE0, timeout, TCP_RTO_MAX);
4105 }
4106 
4107 int tcp_rtx_synack(const struct sock *sk, struct request_sock *req)
4108 {
4109 	const struct tcp_request_sock_ops *af_ops = tcp_rsk(req)->af_specific;
4110 	struct flowi fl;
4111 	int res;
4112 
4113 	/* Paired with WRITE_ONCE() in sock_setsockopt() */
4114 	if (READ_ONCE(sk->sk_txrehash) == SOCK_TXREHASH_ENABLED)
4115 		tcp_rsk(req)->txhash = net_tx_rndhash();
4116 	res = af_ops->send_synack(sk, NULL, &fl, req, NULL, TCP_SYNACK_NORMAL,
4117 				  NULL);
4118 	if (!res) {
4119 		TCP_INC_STATS(sock_net(sk), TCP_MIB_RETRANSSEGS);
4120 		NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPSYNRETRANS);
4121 		if (unlikely(tcp_passive_fastopen(sk)))
4122 			tcp_sk(sk)->total_retrans++;
4123 		trace_tcp_retransmit_synack(sk, req);
4124 	}
4125 	return res;
4126 }
4127 EXPORT_SYMBOL(tcp_rtx_synack);
4128