1 /* SPDX-License-Identifier: GPL-2.0-or-later */
46 #include <linux/bpf-cgroup.h>
59 #define TCP_MIN_GSO_SIZE	(TCP_MIN_SND_MSS - MAX_TCP_OPTION_SPACE)
67 /* Minimal accepted MSS. It is (60+60+8) - (20+20). */
82 /* Maximal number of ACKs sent quickly to accelerate slow-start. */
97 				 * to ~3sec-8min depending on RTO.
104 				 * 15 is ~13-30min depending on RTO.
123 #define TCP_TIMEWAIT_LEN (60*HZ) /* how long to wait to destroy TIME-WAIT
128 				  * to combine FIN-WAIT-2 timeout with
129 				  * TIME-WAIT timer.
133 #define TCP_DELACK_MAX	((unsigned)(HZ/5))	/* maximal time to delay before sending an ACK */
135 #define TCP_DELACK_MIN	((unsigned)(HZ/25))	/* minimal time to delay before sending an ACK */
151 						 * valid RTT sample has been acquired,
168 #define TCP_PAWS_MSL	60		/* Per-host timestamps are invalidated
174 #define TCP_PAWS_WINDOW	1		/* Replay window for per-host
194  * experimental options. See draft-ietf-tcpm-experimental-options-00.txt
223 /* Flags in tp->nonagle */
228 /* TCP thin-stream limits */
265 	if (mem_cgroup_sockets_enabled && sk->sk_memcg &&  in tcp_under_memory_pressure()
266 	    mem_cgroup_under_socket_pressure(sk->sk_memcg))  in tcp_under_memory_pressure()
278         return (__s32)(seq1-seq2) < 0;  in before()
285 	return seq3 - seq2 >= seq1 - seq2;  in between()
290 	if (sk->sk_wmem_queued > SOCK_MIN_SNDBUF &&  in tcp_out_of_memory()
298 	sk_wmem_queued_add(sk, -skb->truesize);  in tcp_wmem_free_skb()
300 		sk_mem_uncharge(sk, skb->truesize);  in tcp_wmem_free_skb()
313 #define TCP_INC_STATS(net, field)	SNMP_INC_STATS((net)->mib.tcp_statistics, field)
314 #define __TCP_INC_STATS(net, field)	__SNMP_INC_STATS((net)->mib.tcp_statistics, field)
315 #define TCP_DEC_STATS(net, field)	SNMP_DEC_STATS((net)->mib.tcp_statistics, field)
316 #define TCP_ADD_STATS(net, field, val)	SNMP_ADD_STATS((net)->mib.tcp_statistics, field, val)
358 	if (icsk->icsk_ack.quick) {  in tcp_dec_quickack_mode()
362 		if (pkts >= icsk->icsk_ack.quick) {  in tcp_dec_quickack_mode()
363 			icsk->icsk_ack.quick = 0;  in tcp_dec_quickack_mode()
365 			icsk->icsk_ack.ato   = TCP_ATO_MIN;  in tcp_dec_quickack_mode()
367 			icsk->icsk_ack.quick -= pkts;  in tcp_dec_quickack_mode()
435  *	BPF SKB-less helpers
514 	if (sk->sk_reuseport) {  in tcp_synq_overflow()
517 		reuse = rcu_dereference(sk->sk_reuseport_cb);  in tcp_synq_overflow()
519 			last_overflow = READ_ONCE(reuse->synq_overflow_ts);  in tcp_synq_overflow()
522 				WRITE_ONCE(reuse->synq_overflow_ts, now);  in tcp_synq_overflow()
527 	last_overflow = READ_ONCE(tcp_sk(sk)->rx_opt.ts_recent_stamp);  in tcp_synq_overflow()
529 		WRITE_ONCE(tcp_sk_rw(sk)->rx_opt.ts_recent_stamp, now);  in tcp_synq_overflow()
538 	if (sk->sk_reuseport) {  in tcp_synq_no_recent_overflow()
541 		reuse = rcu_dereference(sk->sk_reuseport_cb);  in tcp_synq_no_recent_overflow()
543 			last_overflow = READ_ONCE(reuse->synq_overflow_ts);  in tcp_synq_no_recent_overflow()
544 			return !time_between32(now, last_overflow - HZ,  in tcp_synq_no_recent_overflow()
550 	last_overflow = READ_ONCE(tcp_sk(sk)->rx_opt.ts_recent_stamp);  in tcp_synq_no_recent_overflow()
554 	 * 'last_overflow - HZ' as lower bound. That's because a concurrent  in tcp_synq_no_recent_overflow()
559 	return !time_between32(now, last_overflow - HZ,  in tcp_synq_no_recent_overflow()
637 	if (hrtimer_try_to_cancel(&tcp_sk(sk)->pacing_timer) == 1)  in tcp_clear_xmit_timers()
640 	if (hrtimer_try_to_cancel(&tcp_sk(sk)->compressed_ack_timer) == 1)  in tcp_clear_xmit_timers()
656 	 * to sub-MSS pieces for the sake of SWS or making sure there  in tcp_bound_to_half_wnd()
662 	if (tp->max_window > TCP_MSS_DEFAULT)  in tcp_bound_to_half_wnd()
663 		cutoff = (tp->max_window >> 1);  in tcp_bound_to_half_wnd()
665 		cutoff = tp->max_window;  in tcp_bound_to_half_wnd()
668 		return max_t(int, cutoff, 68U - tp->tcp_header_len);  in tcp_bound_to_half_wnd()
676 /* Read 'sendfile()'-style from a TCP socket */
691 	if (inet_csk(sk)->icsk_rto > TCP_RTO_MAX)  in tcp_bound_rto()
692 		inet_csk(sk)->icsk_rto = TCP_RTO_MAX;  in tcp_bound_rto()
697 	return usecs_to_jiffies((tp->srtt_us >> 3) + tp->rttvar_us);  in __tcp_set_rto()
706 	tp->pred_flags = htonl((tp->tcp_header_len << 26) |  in __tcp_fast_path_on()
713 	__tcp_fast_path_on(tp, tp->snd_wnd >> tp->rx_opt.snd_wscale);  in tcp_fast_path_on()
720 	if (RB_EMPTY_ROOT(&tp->out_of_order_queue) &&  in tcp_fast_path_check()
721 	    tp->rcv_wnd &&  in tcp_fast_path_check()
722 	    atomic_read(&sk->sk_rmem_alloc) < sk->sk_rcvbuf &&  in tcp_fast_path_check()
723 	    !tp->urg_data)  in tcp_fast_path_check()
733 	u32 rto_min = inet_csk(sk)->icsk_rto_min;  in tcp_rto_min()
753 	return minmax_get(&tp->rtt_min);  in tcp_min_rtt()
762 	s32 win = tp->rcv_wup + tp->rcv_wnd - tp->rcv_nxt;  in tcp_receive_window()
779  * historically has been the same until linux-4.13.
800 /* This should only be used in contexts where tp->tcp_mstamp is up to date */
803 	return div_u64(tp->tcp_mstamp, USEC_PER_SEC / TCP_TS_HZ);  in tcp_time_stamp()
807 static inline u64 tcp_ns_to_ts(u64 ns)  in tcp_ns_to_ts()  argument
809 	return div_u64(ns, NSEC_PER_SEC / TCP_TS_HZ);  in tcp_ns_to_ts()
822 	return max_t(s64, t1 - t0, 0);  in tcp_stamp_us_delta()
827 	return tcp_ns_to_ts(skb->skb_mstamp_ns);  in tcp_skb_timestamp()
833 	return div_u64(skb->skb_mstamp_ns, NSEC_PER_USEC);  in tcp_skb_timestamp_us()
851  * TCP per-packet control information to the transmission code.
852  * We also store the host-order sequence numbers in here too.
854  * If this grows please adjust skbuff.h:skbuff->cb[xxx] size appropriately.
887 			has_rxtstamp:1,	/* SKB has a RX timestamp	*/
892 #define TCPCB_DELIVERED_CE_MASK ((1U<<20) - 1)
913 #define TCP_SKB_CB(__skb)	((struct tcp_skb_cb *)&((__skb)->cb[0]))
919  * as TCP moves IP6CB into a different location in skb->cb[]
923 	return TCP_SKB_CB(skb)->header.h6.iif;  in tcp_v6_iif()
928 	bool l3_slave = ipv6_l3mdev_skb(TCP_SKB_CB(skb)->header.h6.flags);  in tcp_v6_iif_l3_slave()
930 	return l3_slave ? skb->skb_iif : TCP_SKB_CB(skb)->header.h6.iif;  in tcp_v6_iif_l3_slave()
937 	if (skb && ipv6_l3mdev_skb(TCP_SKB_CB(skb)->header.h6.flags))  in tcp_v6_sdif()
938 		return TCP_SKB_CB(skb)->header.h6.iif;  in tcp_v6_sdif()
955 	if (skb && ipv4_l3mdev_skb(TCP_SKB_CB(skb)->header.h4.flags))  in tcp_v4_sdif()
956 		return TCP_SKB_CB(skb)->header.h4.iif;  in tcp_v4_sdif()
966 	return TCP_SKB_CB(skb)->tcp_gso_segs;  in tcp_skb_pcount()
971 	TCP_SKB_CB(skb)->tcp_gso_segs = segs;  in tcp_skb_pcount_set()
976 	TCP_SKB_CB(skb)->tcp_gso_segs += segs;  in tcp_skb_pcount_add()
982 	return TCP_SKB_CB(skb)->tcp_gso_size;  in tcp_skb_mss()
987 	return likely(!TCP_SKB_CB(skb)->eor);  in tcp_skb_can_collapse_to()
1008 /* Information about inbound ACK, passed to cong_ops->in_ack_event() */
1038 /* A rate sample measures the number of (original/retransmitted) data
1040  * The tcp_rate.c code fills in the rate sample, and congestion
1042  * of ACK processing can optionally chose to consult this sample when
1044  * A sample is invalid if "delivered" or "interval_us" is negative.
1048 	u32  prior_delivered;	/* tp->delivered at "prior_mstamp" */
1049 	u32  prior_delivered_ce;/* tp->delivered_ce at "prior_mstamp" */
1052 	long interval_us;	/* time for tp->delivered to incr "delivered" */
1055 	long rtt_us;		/* RTT of last (S)ACKed packet (or -1) */
1060 	bool is_app_limited;	/* is sample from packet with bubble in pipe? */
1061 	bool is_retrans;	/* is sample from retransmission? */
1084 	void (*pkts_acked)(struct sock *sk, const struct ack_sample *sample);
1157 	return icsk->icsk_ca_ops->flags & TCP_CONG_NEEDS_ECN;  in tcp_ca_needs_ecn()
1164 	if (icsk->icsk_ca_ops->cwnd_event)  in tcp_ca_event()
1165 		icsk->icsk_ca_ops->cwnd_event(sk, event);  in tcp_ca_event()
1188  * tcp_is_sack - SACK enabled
1189  * tcp_is_reno - No SACK
1193 	return likely(tp->rx_opt.sack_ok);  in tcp_is_sack()
1203 	return tp->sacked_out + tp->lost_out;  in tcp_left_out()
1212  * tp->packets_out to determine if the send queue is empty or not.
1222 	return tp->packets_out - tcp_left_out(tp) + tp->retrans_out;  in tcp_packets_in_flight()
1229 	return tp->snd_cwnd;  in tcp_snd_cwnd()
1235 	tp->snd_cwnd = val;  in tcp_snd_cwnd_set()
1240 	return tcp_snd_cwnd(tp) < tp->snd_ssthresh;  in tcp_in_slow_start()
1245 	return tp->snd_ssthresh >= TCP_INFINITE_SSTHRESH;  in tcp_in_initial_slowstart()
1251 	       (1 << inet_csk(sk)->icsk_ca_state);  in tcp_in_cwnd_reduction()
1254 /* If cwnd > ssthresh, we may raise ssthresh to be half-way to cwnd.
1263 		return tp->snd_ssthresh;  in tcp_current_ssthresh()
1265 		return max(tp->snd_ssthresh,  in tcp_current_ssthresh()
1271 #define tcp_verify_left_out(tp)	WARN_ON(tcp_left_out(tp) > tp->packets_out)
1287 	return tp->snd_una + tp->snd_wnd;  in tcp_wnd_end()
1301  * usage, and allow application-limited process to probe bw more aggressively.
1307 	if (tp->is_cwnd_limited)  in tcp_is_cwnd_limited()
1312 		return tcp_snd_cwnd(tp) < 2 * tp->max_packets_out;  in tcp_is_cwnd_limited()
1325 	return smp_load_acquire(&sk->sk_pacing_status) == SK_PACING_NEEDED;  in tcp_needs_internal_pacing()
1333 	s64 delay = tcp_sk(sk)->tcp_wstamp_ns - tcp_sk(sk)->tcp_clock_cache;  in tcp_pacing_delay()  local
1335 	return delay > 0 ? nsecs_to_jiffies(delay) : 0;  in tcp_pacing_delay()
1355 	return max_t(unsigned long, inet_csk(sk)->icsk_rto, TCP_RTO_MIN);  in tcp_probe0_base()
1363 			   inet_csk(sk)->icsk_backoff);  in tcp_probe0_when()
1371 	if (!tcp_sk(sk)->packets_out && !inet_csk(sk)->icsk_pending)  in tcp_check_probe_timer()
1378 	tp->snd_wl1 = seq;  in tcp_init_wl()
1383 	tp->snd_wl1 = seq;  in tcp_update_wl()
1412 	rx_opt->dsack = 0;  in tcp_sack_reset()
1413 	rx_opt->num_sacks = 0;  in tcp_sack_reset()
1420 	const struct tcp_congestion_ops *ca_ops = inet_csk(sk)->icsk_ca_ops;  in tcp_slow_start_after_idle_check()
1424 	if (!READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_slow_start_after_idle) ||  in tcp_slow_start_after_idle_check()
1425 	    tp->packets_out || ca_ops->cong_control)  in tcp_slow_start_after_idle_check()
1427 	delta = tcp_jiffies32 - tp->lsndtime;  in tcp_slow_start_after_idle_check()
1428 	if (delta > inet_csk(sk)->icsk_rto)  in tcp_slow_start_after_idle_check()
1447 	return __tcp_win_from_space(tcp_sk(sk)->scaling_ratio, space);  in tcp_win_from_space()
1461 	return __tcp_space_from_win(tcp_sk(sk)->scaling_ratio, win);  in tcp_space_from_win()
1464 /* Assume a 50% default for skb->len/skb->truesize ratio.
1467 #define TCP_DEFAULT_SCALING_RATIO (1 << (TCP_RMEM_TO_WIN_SCALE - 1))
1471 	tcp_sk(sk)->scaling_ratio = TCP_DEFAULT_SCALING_RATIO;  in tcp_scaling_ratio_init()
1477 	return tcp_win_from_space(sk, READ_ONCE(sk->sk_rcvbuf) -  in tcp_space()
1478 				  READ_ONCE(sk->sk_backlog.len) -  in tcp_space()
1479 				  atomic_read(&sk->sk_rmem_alloc));  in tcp_space()
1484 	return tcp_win_from_space(sk, READ_ONCE(sk->sk_rcvbuf));  in tcp_full_space()
1492 	tp->rcv_ssthresh = min(tp->rcv_ssthresh, new_ssthresh);  in __tcp_adjust_rcv_ssthresh()
1494 		tp->rcv_ssthresh = max_t(u32, tp->rcv_ssthresh,  in __tcp_adjust_rcv_ssthresh()
1500 	__tcp_adjust_rcv_ssthresh(sk, 4U * tcp_sk(sk)->advmss);  in tcp_adjust_rcv_ssthresh()
1519 	rcvbuf = READ_ONCE(sk->sk_rcvbuf);  in tcp_rmem_pressure()
1520 	threshold = rcvbuf - (rcvbuf >> 3);  in tcp_rmem_pressure()
1522 	return atomic_read(&sk->sk_rmem_alloc) > threshold;  in tcp_rmem_pressure()
1528 	int avail = READ_ONCE(tp->rcv_nxt) - READ_ONCE(tp->copied_seq);  in tcp_epollin_ready()
1534 	       (tcp_receive_window(tp) <= inet_csk(sk)->icsk_ack.rcv_mss);  in tcp_epollin_ready()
1552 	val = READ_ONCE(tp->keepalive_intvl);  in keepalive_intvl_when()
1554 	return val ? : READ_ONCE(net->ipv4.sysctl_tcp_keepalive_intvl);  in keepalive_intvl_when()
1563 	val = READ_ONCE(tp->keepalive_time);  in keepalive_time_when()
1565 	return val ? : READ_ONCE(net->ipv4.sysctl_tcp_keepalive_time);  in keepalive_time_when()
1576 	val = READ_ONCE(tp->keepalive_probes);  in keepalive_probes()
1578 	return val ? : READ_ONCE(net->ipv4.sysctl_tcp_keepalive_probes);  in keepalive_probes()
1583 	const struct inet_connection_sock *icsk = &tp->inet_conn;  in keepalive_time_elapsed()
1585 	return min_t(u32, tcp_jiffies32 - icsk->icsk_ack.lrcvtime,  in keepalive_time_elapsed()
1586 			  tcp_jiffies32 - tp->rcv_tstamp);  in keepalive_time_elapsed()
1591 	int fin_timeout = tcp_sk(sk)->linger2 ? :  in tcp_fin_time()
1592 		READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_fin_timeout);  in tcp_fin_time()
1593 	const int rto = inet_csk(sk)->icsk_rto;  in tcp_fin_time()
1595 	if (fin_timeout < (rto << 2) - (rto >> 1))  in tcp_fin_time()
1596 		fin_timeout = (rto << 2) - (rto >> 1);  in tcp_fin_time()
1604 	if ((s32)(rx_opt->ts_recent - rx_opt->rcv_tsval) <= paws_win)  in tcp_paws_check()
1607 				    rx_opt->ts_recent_stamp + TCP_PAWS_24DAYS)))  in tcp_paws_check()
1614 	if (!rx_opt->ts_recent)  in tcp_paws_check()
1630 	   out-of-sync and half-open connections will not be reset.  in tcp_paws_reject()
1633 	   via reboots. Linux-2.2 DOES NOT!  in tcp_paws_reject()
1638 				  rx_opt->ts_recent_stamp + TCP_PAWS_MSL))  in tcp_paws_reject()
1652 	TCP_ADD_STATS(net, TCP_MIB_MAXCONN, -1);  in tcp_mib_init()
1658 	tp->lost_skb_hint = NULL;  in tcp_clear_retrans_hints_partial()
1664 	tp->retransmit_skb_hint = NULL;  in tcp_clear_all_retrans_hints()
1674 /* - key database */
1687 /* - sock block */
1693 /* - pseudo header */
1716 /* - pool: digest algorithm, hash description and scratch buffer */
1722 /* - functions */
1758 #define tcp_twsk_md5_key(twsk)	((twsk)->tw_md5_key)
1843 	ctx = rcu_dereference(inet_csk(sk)->icsk_accept_queue.fastopenq.ctx);  in tcp_fastopen_get_ctx()
1845 		ctx = rcu_dereference(sock_net(sk)->ipv4.tcp_fastopen_ctx);  in tcp_fastopen_get_ctx()
1853 	if (orig->len == TCP_FASTOPEN_COOKIE_SIZE &&  in tcp_fastopen_cookie_match()
1854 	    orig->len == foc->len &&  in tcp_fastopen_cookie_match()
1855 	    !memcmp(orig->val, foc->val, foc->len))  in tcp_fastopen_cookie_match()
1863 	return ctx->num;  in tcp_fastopen_context_len()
1867  * chronograph-like stats that are mutually exclusive.
1871 	TCP_CHRONO_BUSY, /* Actively sending data (non-empty write queue) */
1880 /* This helper is needed, because skb->tcp_tsorted_anchor uses
1881  * the same memory storage than skb->destructor/_skb_refdst
1885 	skb->destructor = NULL;  in tcp_skb_tsorted_anchor_cleanup()
1886 	skb->_skb_refdst = 0UL;  in tcp_skb_tsorted_anchor_cleanup()
1890 	unsigned long _save = skb->_skb_refdst;	\
1891 	skb->_skb_refdst = 0UL;
1894 	skb->_skb_refdst = _save;		\
1901 	return skb_rb_first(&sk->tcp_rtx_queue);  in tcp_rtx_queue_head()
1906 	return skb_rb_last(&sk->tcp_rtx_queue);  in tcp_rtx_queue_tail()
1911 	return skb_peek_tail(&sk->sk_write_queue);  in tcp_write_queue_tail()
1915 	skb_queue_walk_from_safe(&(sk)->sk_write_queue, skb, tmp)
1919 	return skb_peek(&sk->sk_write_queue);  in tcp_send_head()
1925 	return skb_queue_is_last(&sk->sk_write_queue, skb);  in tcp_skb_is_last()
1929  * tcp_write_queue_empty - test if any payload (or FIN) is available in write queue
1933  * we must not use "return skb_queue_empty(&sk->sk_write_queue)"
1939 	return tp->write_seq == tp->snd_nxt;  in tcp_write_queue_empty()
1944 	return RB_EMPTY_ROOT(&sk->tcp_rtx_queue);  in tcp_rtx_queue_empty()
1954 	__skb_queue_tail(&sk->sk_write_queue, skb);  in tcp_add_write_queue_tail()
1957 	if (sk->sk_write_queue.next == skb)  in tcp_add_write_queue_tail()
1966 	__skb_queue_before(&sk->sk_write_queue, skb, new);  in tcp_insert_write_queue_before()
1972 	__skb_unlink(skb, &sk->sk_write_queue);  in tcp_unlink_write_queue()
1980 	rb_erase(&skb->rbnode, &sk->tcp_rtx_queue);  in tcp_rtx_queue_unlink()
1985 	list_del(&skb->tcp_tsorted_anchor);  in tcp_rtx_queue_unlink_and_free()
1995 		__tcp_push_pending_frames(sk, tcp_current_mss(sk), tp->nonagle);  in tcp_push_pending_frames()
2005 	if (!tp->sacked_out)  in tcp_highest_sack_seq()
2006 		return tp->snd_una;  in tcp_highest_sack_seq()
2008 	if (tp->highest_sack == NULL)  in tcp_highest_sack_seq()
2009 		return tp->snd_nxt;  in tcp_highest_sack_seq()
2011 	return TCP_SKB_CB(tp->highest_sack)->seq;  in tcp_highest_sack_seq()
2016 	tcp_sk(sk)->highest_sack = skb_rb_next(skb);  in tcp_advance_highest_sack()
2021 	return tcp_sk(sk)->highest_sack;  in tcp_highest_sack()
2026 	tcp_sk(sk)->highest_sack = tcp_rtx_queue_head(sk);  in tcp_highest_sack_reset()
2035 		tcp_sk(sk)->highest_sack = new;  in tcp_highest_sack_replace()
2041 	switch (sk->sk_state) {  in inet_sk_transparent()
2043 		return inet_twsk(sk)->tw_transparent;  in inet_sk_transparent()
2045 		return inet_rsk(inet_reqsk(sk))->no_srccheck;  in inet_sk_transparent()
2051  * increased latency). Used to trigger latency-reducing mechanisms.
2055 	return tp->packets_out < 4 && !tcp_in_initial_slowstart(tp);  in tcp_stream_is_thin()
2101 	val = READ_ONCE(tp->notsent_lowat);  in tcp_notsent_lowat()
2103 	return val ?: READ_ONCE(net->ipv4.sysctl_tcp_notsent_lowat);  in tcp_notsent_lowat()
2118 /* TCP af-specific functions */
2173 	return ops->cookie_init_seq(skb, mss);  in cookie_init_sequence()
2220 	plb->consec_cong_rounds = 0;  in tcp_plb_init()
2221 	plb->pause_until = 0;  in tcp_plb_init()
2232 	u32 rto = inet_csk(sk)->icsk_rto;  in tcp_rto_delta_us()
2237 		return rto_time_stamp_us - tcp_sk(sk)->tcp_mstamp;  in tcp_rto_delta_us()
2244 			tcp_sk(sk)->packets_out, tcp_sk(sk)->sacked_out,  in tcp_rto_delta_us()
2245 			tcp_sk(sk)->lost_out, tcp_sk(sk)->retrans_out,  in tcp_rto_delta_us()
2246 			tcp_sk(sk)->tlp_high_seq, sk->sk_state,  in tcp_rto_delta_us()
2247 			inet_csk(sk)->icsk_ca_state,  in tcp_rto_delta_us()
2248 			tcp_sk(sk)->advmss, tcp_sk(sk)->mss_cache,  in tcp_rto_delta_us()
2249 			inet_csk(sk)->icsk_pmtu_cookie);  in tcp_rto_delta_us()
2261 	const struct ip_options *opt = &TCP_SKB_CB(skb)->header.h4.opt;  in tcp_v4_save_options()
2264 	if (opt->optlen) {  in tcp_v4_save_options()
2265 		int opt_size = sizeof(*dopt) + opt->optlen;  in tcp_v4_save_options()
2268 		if (dopt && __ip_options_echo(net, &dopt->opt, skb, opt)) {  in tcp_v4_save_options()
2276 /* locally generated TCP pure ACKs have skb->truesize == 2
2283 	return skb->truesize == 2;  in skb_is_tcp_pure_ack()
2288 	skb->truesize = 2;  in skb_set_tcp_pure_ack()
2296 	if ((1 << sk->sk_state) & (TCPF_SYN_SENT | TCPF_SYN_RECV)) {  in tcp_inq()
2299 		   !tp->urg_data ||  in tcp_inq()
2300 		   before(tp->urg_seq, tp->copied_seq) ||  in tcp_inq()
2301 		   !before(tp->urg_seq, tp->rcv_nxt)) {  in tcp_inq()
2303 		answ = tp->rcv_nxt - tp->copied_seq;  in tcp_inq()
2307 			answ--;  in tcp_inq()
2309 		answ = tp->urg_seq - tp->copied_seq;  in tcp_inq()
2321 	segs_in = max_t(u16, 1, skb_shinfo(skb)->gso_segs);  in tcp_segs_in()
2326 	WRITE_ONCE(tp->segs_in, tp->segs_in + segs_in);  in tcp_segs_in()
2327 	if (skb->len > tcp_hdrlen(skb))  in tcp_segs_in()
2328 		WRITE_ONCE(tp->data_segs_in, tp->data_segs_in + segs_in);  in tcp_segs_in()
2340 	atomic_inc(&((struct sock *)sk)->sk_drops);  in tcp_listendrop()
2384 	__MODULE_INFO(alias, alias_tcp_ulp, "tcp-ulp-" name)
2418 	skops->skb = skb;  in bpf_skops_init_skb()
2419 	skops->skb_data_end = skb->data + end_offset;  in bpf_skops_init_skb()
2455 		ret = -1;  in tcp_call_bpf()
2477 	return -EPERM;  in tcp_call_bpf()
2482 	return -EPERM;  in tcp_call_bpf_2arg()
2488 	return -EPERM;  in tcp_call_bpf_3arg()
2542 		skb->skb_mstamp_ns += (u64)tp->tcp_tx_delay * NSEC_PER_USEC;  in tcp_add_tx_delay()
2551 		u32 delay = (sk->sk_state == TCP_TIME_WAIT) ?  in tcp_transmit_time()  local
2552 			tcp_twsk(sk)->tw_tx_delay : tcp_sk(sk)->tcp_tx_delay;  in tcp_transmit_time()
2554 		return tcp_clock_ns() + (u64)delay * NSEC_PER_USEC;  in tcp_transmit_time()