xref: /openbmc/linux/net/ipv4/tcp_ipv4.c (revision fb8d6c8d)
1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /*
3  * INET		An implementation of the TCP/IP protocol suite for the LINUX
4  *		operating system.  INET is implemented using the  BSD Socket
5  *		interface as the means of communication with the user level.
6  *
7  *		Implementation of the Transmission Control Protocol(TCP).
8  *
9  *		IPv4 specific functions
10  *
11  *		code split from:
12  *		linux/ipv4/tcp.c
13  *		linux/ipv4/tcp_input.c
14  *		linux/ipv4/tcp_output.c
15  *
16  *		See tcp.c for author information
17  */
18 
19 /*
20  * Changes:
21  *		David S. Miller	:	New socket lookup architecture.
22  *					This code is dedicated to John Dyson.
23  *		David S. Miller :	Change semantics of established hash,
24  *					half is devoted to TIME_WAIT sockets
25  *					and the rest go in the other half.
26  *		Andi Kleen :		Add support for syncookies and fixed
27  *					some bugs: ip options weren't passed to
28  *					the TCP layer, missed a check for an
29  *					ACK bit.
30  *		Andi Kleen :		Implemented fast path mtu discovery.
31  *	     				Fixed many serious bugs in the
32  *					request_sock handling and moved
33  *					most of it into the af independent code.
34  *					Added tail drop and some other bugfixes.
35  *					Added new listen semantics.
36  *		Mike McLagan	:	Routing by source
37  *	Juan Jose Ciarlante:		ip_dynaddr bits
38  *		Andi Kleen:		various fixes.
39  *	Vitaly E. Lavrov	:	Transparent proxy revived after year
40  *					coma.
41  *	Andi Kleen		:	Fix new listen.
42  *	Andi Kleen		:	Fix accept error reporting.
43  *	YOSHIFUJI Hideaki @USAGI and:	Support IPV6_V6ONLY socket option, which
44  *	Alexey Kuznetsov		allow both IPv4 and IPv6 sockets to bind
45  *					a single port at the same time.
46  */
47 
48 #define pr_fmt(fmt) "TCP: " fmt
49 
50 #include <linux/bottom_half.h>
51 #include <linux/types.h>
52 #include <linux/fcntl.h>
53 #include <linux/module.h>
54 #include <linux/random.h>
55 #include <linux/cache.h>
56 #include <linux/jhash.h>
57 #include <linux/init.h>
58 #include <linux/times.h>
59 #include <linux/slab.h>
60 
61 #include <net/net_namespace.h>
62 #include <net/icmp.h>
63 #include <net/inet_hashtables.h>
64 #include <net/tcp.h>
65 #include <net/transp_v6.h>
66 #include <net/ipv6.h>
67 #include <net/inet_common.h>
68 #include <net/timewait_sock.h>
69 #include <net/xfrm.h>
70 #include <net/secure_seq.h>
71 #include <net/busy_poll.h>
72 
73 #include <linux/inet.h>
74 #include <linux/ipv6.h>
75 #include <linux/stddef.h>
76 #include <linux/proc_fs.h>
77 #include <linux/seq_file.h>
78 #include <linux/inetdevice.h>
79 
80 #include <crypto/hash.h>
81 #include <linux/scatterlist.h>
82 
83 #include <trace/events/tcp.h>
84 
85 #ifdef CONFIG_TCP_MD5SIG
86 static int tcp_v4_md5_hash_hdr(char *md5_hash, const struct tcp_md5sig_key *key,
87 			       __be32 daddr, __be32 saddr, const struct tcphdr *th);
88 #endif
89 
90 struct inet_hashinfo tcp_hashinfo;
91 EXPORT_SYMBOL(tcp_hashinfo);
92 
93 static u32 tcp_v4_init_seq(const struct sk_buff *skb)
94 {
95 	return secure_tcp_seq(ip_hdr(skb)->daddr,
96 			      ip_hdr(skb)->saddr,
97 			      tcp_hdr(skb)->dest,
98 			      tcp_hdr(skb)->source);
99 }
100 
101 static u32 tcp_v4_init_ts_off(const struct net *net, const struct sk_buff *skb)
102 {
103 	return secure_tcp_ts_off(net, ip_hdr(skb)->daddr, ip_hdr(skb)->saddr);
104 }
105 
106 int tcp_twsk_unique(struct sock *sk, struct sock *sktw, void *twp)
107 {
108 	const struct inet_timewait_sock *tw = inet_twsk(sktw);
109 	const struct tcp_timewait_sock *tcptw = tcp_twsk(sktw);
110 	struct tcp_sock *tp = tcp_sk(sk);
111 	int reuse = sock_net(sk)->ipv4.sysctl_tcp_tw_reuse;
112 
113 	if (reuse == 2) {
114 		/* Still does not detect *everything* that goes through
115 		 * lo, since we require a loopback src or dst address
116 		 * or direct binding to 'lo' interface.
117 		 */
118 		bool loopback = false;
119 		if (tw->tw_bound_dev_if == LOOPBACK_IFINDEX)
120 			loopback = true;
121 #if IS_ENABLED(CONFIG_IPV6)
122 		if (tw->tw_family == AF_INET6) {
123 			if (ipv6_addr_loopback(&tw->tw_v6_daddr) ||
124 			    (ipv6_addr_v4mapped(&tw->tw_v6_daddr) &&
125 			     (tw->tw_v6_daddr.s6_addr[12] == 127)) ||
126 			    ipv6_addr_loopback(&tw->tw_v6_rcv_saddr) ||
127 			    (ipv6_addr_v4mapped(&tw->tw_v6_rcv_saddr) &&
128 			     (tw->tw_v6_rcv_saddr.s6_addr[12] == 127)))
129 				loopback = true;
130 		} else
131 #endif
132 		{
133 			if (ipv4_is_loopback(tw->tw_daddr) ||
134 			    ipv4_is_loopback(tw->tw_rcv_saddr))
135 				loopback = true;
136 		}
137 		if (!loopback)
138 			reuse = 0;
139 	}
140 
141 	/* With PAWS, it is safe from the viewpoint
142 	   of data integrity. Even without PAWS it is safe provided sequence
143 	   spaces do not overlap i.e. at data rates <= 80Mbit/sec.
144 
145 	   Actually, the idea is close to VJ's one, only timestamp cache is
146 	   held not per host, but per port pair and TW bucket is used as state
147 	   holder.
148 
149 	   If TW bucket has been already destroyed we fall back to VJ's scheme
150 	   and use initial timestamp retrieved from peer table.
151 	 */
152 	if (tcptw->tw_ts_recent_stamp &&
153 	    (!twp || (reuse && time_after32(ktime_get_seconds(),
154 					    tcptw->tw_ts_recent_stamp)))) {
155 		/* In case of repair and re-using TIME-WAIT sockets we still
156 		 * want to be sure that it is safe as above but honor the
157 		 * sequence numbers and time stamps set as part of the repair
158 		 * process.
159 		 *
160 		 * Without this check re-using a TIME-WAIT socket with TCP
161 		 * repair would accumulate a -1 on the repair assigned
162 		 * sequence number. The first time it is reused the sequence
163 		 * is -1, the second time -2, etc. This fixes that issue
164 		 * without appearing to create any others.
165 		 */
166 		if (likely(!tp->repair)) {
167 			u32 seq = tcptw->tw_snd_nxt + 65535 + 2;
168 
169 			if (!seq)
170 				seq = 1;
171 			WRITE_ONCE(tp->write_seq, seq);
172 			tp->rx_opt.ts_recent	   = tcptw->tw_ts_recent;
173 			tp->rx_opt.ts_recent_stamp = tcptw->tw_ts_recent_stamp;
174 		}
175 		sock_hold(sktw);
176 		return 1;
177 	}
178 
179 	return 0;
180 }
181 EXPORT_SYMBOL_GPL(tcp_twsk_unique);
182 
183 static int tcp_v4_pre_connect(struct sock *sk, struct sockaddr *uaddr,
184 			      int addr_len)
185 {
186 	/* This check is replicated from tcp_v4_connect() and intended to
187 	 * prevent BPF program called below from accessing bytes that are out
188 	 * of the bound specified by user in addr_len.
189 	 */
190 	if (addr_len < sizeof(struct sockaddr_in))
191 		return -EINVAL;
192 
193 	sock_owned_by_me(sk);
194 
195 	return BPF_CGROUP_RUN_PROG_INET4_CONNECT(sk, uaddr);
196 }
197 
198 /* This will initiate an outgoing connection. */
199 int tcp_v4_connect(struct sock *sk, struct sockaddr *uaddr, int addr_len)
200 {
201 	struct sockaddr_in *usin = (struct sockaddr_in *)uaddr;
202 	struct inet_sock *inet = inet_sk(sk);
203 	struct tcp_sock *tp = tcp_sk(sk);
204 	__be16 orig_sport, orig_dport;
205 	__be32 daddr, nexthop;
206 	struct flowi4 *fl4;
207 	struct rtable *rt;
208 	int err;
209 	struct ip_options_rcu *inet_opt;
210 	struct inet_timewait_death_row *tcp_death_row = &sock_net(sk)->ipv4.tcp_death_row;
211 
212 	if (addr_len < sizeof(struct sockaddr_in))
213 		return -EINVAL;
214 
215 	if (usin->sin_family != AF_INET)
216 		return -EAFNOSUPPORT;
217 
218 	nexthop = daddr = usin->sin_addr.s_addr;
219 	inet_opt = rcu_dereference_protected(inet->inet_opt,
220 					     lockdep_sock_is_held(sk));
221 	if (inet_opt && inet_opt->opt.srr) {
222 		if (!daddr)
223 			return -EINVAL;
224 		nexthop = inet_opt->opt.faddr;
225 	}
226 
227 	orig_sport = inet->inet_sport;
228 	orig_dport = usin->sin_port;
229 	fl4 = &inet->cork.fl.u.ip4;
230 	rt = ip_route_connect(fl4, nexthop, inet->inet_saddr,
231 			      RT_CONN_FLAGS(sk), sk->sk_bound_dev_if,
232 			      IPPROTO_TCP,
233 			      orig_sport, orig_dport, sk);
234 	if (IS_ERR(rt)) {
235 		err = PTR_ERR(rt);
236 		if (err == -ENETUNREACH)
237 			IP_INC_STATS(sock_net(sk), IPSTATS_MIB_OUTNOROUTES);
238 		return err;
239 	}
240 
241 	if (rt->rt_flags & (RTCF_MULTICAST | RTCF_BROADCAST)) {
242 		ip_rt_put(rt);
243 		return -ENETUNREACH;
244 	}
245 
246 	if (!inet_opt || !inet_opt->opt.srr)
247 		daddr = fl4->daddr;
248 
249 	if (!inet->inet_saddr)
250 		inet->inet_saddr = fl4->saddr;
251 	sk_rcv_saddr_set(sk, inet->inet_saddr);
252 
253 	if (tp->rx_opt.ts_recent_stamp && inet->inet_daddr != daddr) {
254 		/* Reset inherited state */
255 		tp->rx_opt.ts_recent	   = 0;
256 		tp->rx_opt.ts_recent_stamp = 0;
257 		if (likely(!tp->repair))
258 			WRITE_ONCE(tp->write_seq, 0);
259 	}
260 
261 	inet->inet_dport = usin->sin_port;
262 	sk_daddr_set(sk, daddr);
263 
264 	inet_csk(sk)->icsk_ext_hdr_len = 0;
265 	if (inet_opt)
266 		inet_csk(sk)->icsk_ext_hdr_len = inet_opt->opt.optlen;
267 
268 	tp->rx_opt.mss_clamp = TCP_MSS_DEFAULT;
269 
270 	/* Socket identity is still unknown (sport may be zero).
271 	 * However we set state to SYN-SENT and not releasing socket
272 	 * lock select source port, enter ourselves into the hash tables and
273 	 * complete initialization after this.
274 	 */
275 	tcp_set_state(sk, TCP_SYN_SENT);
276 	err = inet_hash_connect(tcp_death_row, sk);
277 	if (err)
278 		goto failure;
279 
280 	sk_set_txhash(sk);
281 
282 	rt = ip_route_newports(fl4, rt, orig_sport, orig_dport,
283 			       inet->inet_sport, inet->inet_dport, sk);
284 	if (IS_ERR(rt)) {
285 		err = PTR_ERR(rt);
286 		rt = NULL;
287 		goto failure;
288 	}
289 	/* OK, now commit destination to socket.  */
290 	sk->sk_gso_type = SKB_GSO_TCPV4;
291 	sk_setup_caps(sk, &rt->dst);
292 	rt = NULL;
293 
294 	if (likely(!tp->repair)) {
295 		if (!tp->write_seq)
296 			WRITE_ONCE(tp->write_seq,
297 				   secure_tcp_seq(inet->inet_saddr,
298 						  inet->inet_daddr,
299 						  inet->inet_sport,
300 						  usin->sin_port));
301 		tp->tsoffset = secure_tcp_ts_off(sock_net(sk),
302 						 inet->inet_saddr,
303 						 inet->inet_daddr);
304 	}
305 
306 	inet->inet_id = prandom_u32();
307 
308 	if (tcp_fastopen_defer_connect(sk, &err))
309 		return err;
310 	if (err)
311 		goto failure;
312 
313 	err = tcp_connect(sk);
314 
315 	if (err)
316 		goto failure;
317 
318 	return 0;
319 
320 failure:
321 	/*
322 	 * This unhashes the socket and releases the local port,
323 	 * if necessary.
324 	 */
325 	tcp_set_state(sk, TCP_CLOSE);
326 	ip_rt_put(rt);
327 	sk->sk_route_caps = 0;
328 	inet->inet_dport = 0;
329 	return err;
330 }
331 EXPORT_SYMBOL(tcp_v4_connect);
332 
333 /*
334  * This routine reacts to ICMP_FRAG_NEEDED mtu indications as defined in RFC1191.
335  * It can be called through tcp_release_cb() if socket was owned by user
336  * at the time tcp_v4_err() was called to handle ICMP message.
337  */
338 void tcp_v4_mtu_reduced(struct sock *sk)
339 {
340 	struct inet_sock *inet = inet_sk(sk);
341 	struct dst_entry *dst;
342 	u32 mtu;
343 
344 	if ((1 << sk->sk_state) & (TCPF_LISTEN | TCPF_CLOSE))
345 		return;
346 	mtu = tcp_sk(sk)->mtu_info;
347 	dst = inet_csk_update_pmtu(sk, mtu);
348 	if (!dst)
349 		return;
350 
351 	/* Something is about to be wrong... Remember soft error
352 	 * for the case, if this connection will not able to recover.
353 	 */
354 	if (mtu < dst_mtu(dst) && ip_dont_fragment(sk, dst))
355 		sk->sk_err_soft = EMSGSIZE;
356 
357 	mtu = dst_mtu(dst);
358 
359 	if (inet->pmtudisc != IP_PMTUDISC_DONT &&
360 	    ip_sk_accept_pmtu(sk) &&
361 	    inet_csk(sk)->icsk_pmtu_cookie > mtu) {
362 		tcp_sync_mss(sk, mtu);
363 
364 		/* Resend the TCP packet because it's
365 		 * clear that the old packet has been
366 		 * dropped. This is the new "fast" path mtu
367 		 * discovery.
368 		 */
369 		tcp_simple_retransmit(sk);
370 	} /* else let the usual retransmit timer handle it */
371 }
372 EXPORT_SYMBOL(tcp_v4_mtu_reduced);
373 
374 static void do_redirect(struct sk_buff *skb, struct sock *sk)
375 {
376 	struct dst_entry *dst = __sk_dst_check(sk, 0);
377 
378 	if (dst)
379 		dst->ops->redirect(dst, sk, skb);
380 }
381 
382 
383 /* handle ICMP messages on TCP_NEW_SYN_RECV request sockets */
384 void tcp_req_err(struct sock *sk, u32 seq, bool abort)
385 {
386 	struct request_sock *req = inet_reqsk(sk);
387 	struct net *net = sock_net(sk);
388 
389 	/* ICMPs are not backlogged, hence we cannot get
390 	 * an established socket here.
391 	 */
392 	if (seq != tcp_rsk(req)->snt_isn) {
393 		__NET_INC_STATS(net, LINUX_MIB_OUTOFWINDOWICMPS);
394 	} else if (abort) {
395 		/*
396 		 * Still in SYN_RECV, just remove it silently.
397 		 * There is no good way to pass the error to the newly
398 		 * created socket, and POSIX does not want network
399 		 * errors returned from accept().
400 		 */
401 		inet_csk_reqsk_queue_drop(req->rsk_listener, req);
402 		tcp_listendrop(req->rsk_listener);
403 	}
404 	reqsk_put(req);
405 }
406 EXPORT_SYMBOL(tcp_req_err);
407 
408 /*
409  * This routine is called by the ICMP module when it gets some
410  * sort of error condition.  If err < 0 then the socket should
411  * be closed and the error returned to the user.  If err > 0
412  * it's just the icmp type << 8 | icmp code.  After adjustment
413  * header points to the first 8 bytes of the tcp header.  We need
414  * to find the appropriate port.
415  *
416  * The locking strategy used here is very "optimistic". When
417  * someone else accesses the socket the ICMP is just dropped
418  * and for some paths there is no check at all.
419  * A more general error queue to queue errors for later handling
420  * is probably better.
421  *
422  */
423 
424 int tcp_v4_err(struct sk_buff *icmp_skb, u32 info)
425 {
426 	const struct iphdr *iph = (const struct iphdr *)icmp_skb->data;
427 	struct tcphdr *th = (struct tcphdr *)(icmp_skb->data + (iph->ihl << 2));
428 	struct inet_connection_sock *icsk;
429 	struct tcp_sock *tp;
430 	struct inet_sock *inet;
431 	const int type = icmp_hdr(icmp_skb)->type;
432 	const int code = icmp_hdr(icmp_skb)->code;
433 	struct sock *sk;
434 	struct sk_buff *skb;
435 	struct request_sock *fastopen;
436 	u32 seq, snd_una;
437 	s32 remaining;
438 	u32 delta_us;
439 	int err;
440 	struct net *net = dev_net(icmp_skb->dev);
441 
442 	sk = __inet_lookup_established(net, &tcp_hashinfo, iph->daddr,
443 				       th->dest, iph->saddr, ntohs(th->source),
444 				       inet_iif(icmp_skb), 0);
445 	if (!sk) {
446 		__ICMP_INC_STATS(net, ICMP_MIB_INERRORS);
447 		return -ENOENT;
448 	}
449 	if (sk->sk_state == TCP_TIME_WAIT) {
450 		inet_twsk_put(inet_twsk(sk));
451 		return 0;
452 	}
453 	seq = ntohl(th->seq);
454 	if (sk->sk_state == TCP_NEW_SYN_RECV) {
455 		tcp_req_err(sk, seq, type == ICMP_PARAMETERPROB ||
456 				     type == ICMP_TIME_EXCEEDED ||
457 				     (type == ICMP_DEST_UNREACH &&
458 				      (code == ICMP_NET_UNREACH ||
459 				       code == ICMP_HOST_UNREACH)));
460 		return 0;
461 	}
462 
463 	bh_lock_sock(sk);
464 	/* If too many ICMPs get dropped on busy
465 	 * servers this needs to be solved differently.
466 	 * We do take care of PMTU discovery (RFC1191) special case :
467 	 * we can receive locally generated ICMP messages while socket is held.
468 	 */
469 	if (sock_owned_by_user(sk)) {
470 		if (!(type == ICMP_DEST_UNREACH && code == ICMP_FRAG_NEEDED))
471 			__NET_INC_STATS(net, LINUX_MIB_LOCKDROPPEDICMPS);
472 	}
473 	if (sk->sk_state == TCP_CLOSE)
474 		goto out;
475 
476 	if (unlikely(iph->ttl < inet_sk(sk)->min_ttl)) {
477 		__NET_INC_STATS(net, LINUX_MIB_TCPMINTTLDROP);
478 		goto out;
479 	}
480 
481 	icsk = inet_csk(sk);
482 	tp = tcp_sk(sk);
483 	/* XXX (TFO) - tp->snd_una should be ISN (tcp_create_openreq_child() */
484 	fastopen = rcu_dereference(tp->fastopen_rsk);
485 	snd_una = fastopen ? tcp_rsk(fastopen)->snt_isn : tp->snd_una;
486 	if (sk->sk_state != TCP_LISTEN &&
487 	    !between(seq, snd_una, tp->snd_nxt)) {
488 		__NET_INC_STATS(net, LINUX_MIB_OUTOFWINDOWICMPS);
489 		goto out;
490 	}
491 
492 	switch (type) {
493 	case ICMP_REDIRECT:
494 		if (!sock_owned_by_user(sk))
495 			do_redirect(icmp_skb, sk);
496 		goto out;
497 	case ICMP_SOURCE_QUENCH:
498 		/* Just silently ignore these. */
499 		goto out;
500 	case ICMP_PARAMETERPROB:
501 		err = EPROTO;
502 		break;
503 	case ICMP_DEST_UNREACH:
504 		if (code > NR_ICMP_UNREACH)
505 			goto out;
506 
507 		if (code == ICMP_FRAG_NEEDED) { /* PMTU discovery (RFC1191) */
508 			/* We are not interested in TCP_LISTEN and open_requests
509 			 * (SYN-ACKs send out by Linux are always <576bytes so
510 			 * they should go through unfragmented).
511 			 */
512 			if (sk->sk_state == TCP_LISTEN)
513 				goto out;
514 
515 			tp->mtu_info = info;
516 			if (!sock_owned_by_user(sk)) {
517 				tcp_v4_mtu_reduced(sk);
518 			} else {
519 				if (!test_and_set_bit(TCP_MTU_REDUCED_DEFERRED, &sk->sk_tsq_flags))
520 					sock_hold(sk);
521 			}
522 			goto out;
523 		}
524 
525 		err = icmp_err_convert[code].errno;
526 		/* check if icmp_skb allows revert of backoff
527 		 * (see draft-zimmermann-tcp-lcd) */
528 		if (code != ICMP_NET_UNREACH && code != ICMP_HOST_UNREACH)
529 			break;
530 		if (seq != tp->snd_una  || !icsk->icsk_retransmits ||
531 		    !icsk->icsk_backoff || fastopen)
532 			break;
533 
534 		if (sock_owned_by_user(sk))
535 			break;
536 
537 		skb = tcp_rtx_queue_head(sk);
538 		if (WARN_ON_ONCE(!skb))
539 			break;
540 
541 		icsk->icsk_backoff--;
542 		icsk->icsk_rto = tp->srtt_us ? __tcp_set_rto(tp) :
543 					       TCP_TIMEOUT_INIT;
544 		icsk->icsk_rto = inet_csk_rto_backoff(icsk, TCP_RTO_MAX);
545 
546 
547 		tcp_mstamp_refresh(tp);
548 		delta_us = (u32)(tp->tcp_mstamp - tcp_skb_timestamp_us(skb));
549 		remaining = icsk->icsk_rto -
550 			    usecs_to_jiffies(delta_us);
551 
552 		if (remaining > 0) {
553 			inet_csk_reset_xmit_timer(sk, ICSK_TIME_RETRANS,
554 						  remaining, TCP_RTO_MAX);
555 		} else {
556 			/* RTO revert clocked out retransmission.
557 			 * Will retransmit now */
558 			tcp_retransmit_timer(sk);
559 		}
560 
561 		break;
562 	case ICMP_TIME_EXCEEDED:
563 		err = EHOSTUNREACH;
564 		break;
565 	default:
566 		goto out;
567 	}
568 
569 	switch (sk->sk_state) {
570 	case TCP_SYN_SENT:
571 	case TCP_SYN_RECV:
572 		/* Only in fast or simultaneous open. If a fast open socket is
573 		 * is already accepted it is treated as a connected one below.
574 		 */
575 		if (fastopen && !fastopen->sk)
576 			break;
577 
578 		if (!sock_owned_by_user(sk)) {
579 			sk->sk_err = err;
580 
581 			sk->sk_error_report(sk);
582 
583 			tcp_done(sk);
584 		} else {
585 			sk->sk_err_soft = err;
586 		}
587 		goto out;
588 	}
589 
590 	/* If we've already connected we will keep trying
591 	 * until we time out, or the user gives up.
592 	 *
593 	 * rfc1122 4.2.3.9 allows to consider as hard errors
594 	 * only PROTO_UNREACH and PORT_UNREACH (well, FRAG_FAILED too,
595 	 * but it is obsoleted by pmtu discovery).
596 	 *
597 	 * Note, that in modern internet, where routing is unreliable
598 	 * and in each dark corner broken firewalls sit, sending random
599 	 * errors ordered by their masters even this two messages finally lose
600 	 * their original sense (even Linux sends invalid PORT_UNREACHs)
601 	 *
602 	 * Now we are in compliance with RFCs.
603 	 *							--ANK (980905)
604 	 */
605 
606 	inet = inet_sk(sk);
607 	if (!sock_owned_by_user(sk) && inet->recverr) {
608 		sk->sk_err = err;
609 		sk->sk_error_report(sk);
610 	} else	{ /* Only an error on timeout */
611 		sk->sk_err_soft = err;
612 	}
613 
614 out:
615 	bh_unlock_sock(sk);
616 	sock_put(sk);
617 	return 0;
618 }
619 
620 void __tcp_v4_send_check(struct sk_buff *skb, __be32 saddr, __be32 daddr)
621 {
622 	struct tcphdr *th = tcp_hdr(skb);
623 
624 	th->check = ~tcp_v4_check(skb->len, saddr, daddr, 0);
625 	skb->csum_start = skb_transport_header(skb) - skb->head;
626 	skb->csum_offset = offsetof(struct tcphdr, check);
627 }
628 
629 /* This routine computes an IPv4 TCP checksum. */
630 void tcp_v4_send_check(struct sock *sk, struct sk_buff *skb)
631 {
632 	const struct inet_sock *inet = inet_sk(sk);
633 
634 	__tcp_v4_send_check(skb, inet->inet_saddr, inet->inet_daddr);
635 }
636 EXPORT_SYMBOL(tcp_v4_send_check);
637 
638 /*
639  *	This routine will send an RST to the other tcp.
640  *
641  *	Someone asks: why I NEVER use socket parameters (TOS, TTL etc.)
642  *		      for reset.
643  *	Answer: if a packet caused RST, it is not for a socket
644  *		existing in our system, if it is matched to a socket,
645  *		it is just duplicate segment or bug in other side's TCP.
646  *		So that we build reply only basing on parameters
647  *		arrived with segment.
648  *	Exception: precedence violation. We do not implement it in any case.
649  */
650 
651 static void tcp_v4_send_reset(const struct sock *sk, struct sk_buff *skb)
652 {
653 	const struct tcphdr *th = tcp_hdr(skb);
654 	struct {
655 		struct tcphdr th;
656 #ifdef CONFIG_TCP_MD5SIG
657 		__be32 opt[(TCPOLEN_MD5SIG_ALIGNED >> 2)];
658 #endif
659 	} rep;
660 	struct ip_reply_arg arg;
661 #ifdef CONFIG_TCP_MD5SIG
662 	struct tcp_md5sig_key *key = NULL;
663 	const __u8 *hash_location = NULL;
664 	unsigned char newhash[16];
665 	int genhash;
666 	struct sock *sk1 = NULL;
667 #endif
668 	u64 transmit_time = 0;
669 	struct sock *ctl_sk;
670 	struct net *net;
671 
672 	/* Never send a reset in response to a reset. */
673 	if (th->rst)
674 		return;
675 
676 	/* If sk not NULL, it means we did a successful lookup and incoming
677 	 * route had to be correct. prequeue might have dropped our dst.
678 	 */
679 	if (!sk && skb_rtable(skb)->rt_type != RTN_LOCAL)
680 		return;
681 
682 	/* Swap the send and the receive. */
683 	memset(&rep, 0, sizeof(rep));
684 	rep.th.dest   = th->source;
685 	rep.th.source = th->dest;
686 	rep.th.doff   = sizeof(struct tcphdr) / 4;
687 	rep.th.rst    = 1;
688 
689 	if (th->ack) {
690 		rep.th.seq = th->ack_seq;
691 	} else {
692 		rep.th.ack = 1;
693 		rep.th.ack_seq = htonl(ntohl(th->seq) + th->syn + th->fin +
694 				       skb->len - (th->doff << 2));
695 	}
696 
697 	memset(&arg, 0, sizeof(arg));
698 	arg.iov[0].iov_base = (unsigned char *)&rep;
699 	arg.iov[0].iov_len  = sizeof(rep.th);
700 
701 	net = sk ? sock_net(sk) : dev_net(skb_dst(skb)->dev);
702 #ifdef CONFIG_TCP_MD5SIG
703 	rcu_read_lock();
704 	hash_location = tcp_parse_md5sig_option(th);
705 	if (sk && sk_fullsock(sk)) {
706 		key = tcp_md5_do_lookup(sk, (union tcp_md5_addr *)
707 					&ip_hdr(skb)->saddr, AF_INET);
708 	} else if (hash_location) {
709 		/*
710 		 * active side is lost. Try to find listening socket through
711 		 * source port, and then find md5 key through listening socket.
712 		 * we are not loose security here:
713 		 * Incoming packet is checked with md5 hash with finding key,
714 		 * no RST generated if md5 hash doesn't match.
715 		 */
716 		sk1 = __inet_lookup_listener(net, &tcp_hashinfo, NULL, 0,
717 					     ip_hdr(skb)->saddr,
718 					     th->source, ip_hdr(skb)->daddr,
719 					     ntohs(th->source), inet_iif(skb),
720 					     tcp_v4_sdif(skb));
721 		/* don't send rst if it can't find key */
722 		if (!sk1)
723 			goto out;
724 
725 		key = tcp_md5_do_lookup(sk1, (union tcp_md5_addr *)
726 					&ip_hdr(skb)->saddr, AF_INET);
727 		if (!key)
728 			goto out;
729 
730 
731 		genhash = tcp_v4_md5_hash_skb(newhash, key, NULL, skb);
732 		if (genhash || memcmp(hash_location, newhash, 16) != 0)
733 			goto out;
734 
735 	}
736 
737 	if (key) {
738 		rep.opt[0] = htonl((TCPOPT_NOP << 24) |
739 				   (TCPOPT_NOP << 16) |
740 				   (TCPOPT_MD5SIG << 8) |
741 				   TCPOLEN_MD5SIG);
742 		/* Update length and the length the header thinks exists */
743 		arg.iov[0].iov_len += TCPOLEN_MD5SIG_ALIGNED;
744 		rep.th.doff = arg.iov[0].iov_len / 4;
745 
746 		tcp_v4_md5_hash_hdr((__u8 *) &rep.opt[1],
747 				     key, ip_hdr(skb)->saddr,
748 				     ip_hdr(skb)->daddr, &rep.th);
749 	}
750 #endif
751 	arg.csum = csum_tcpudp_nofold(ip_hdr(skb)->daddr,
752 				      ip_hdr(skb)->saddr, /* XXX */
753 				      arg.iov[0].iov_len, IPPROTO_TCP, 0);
754 	arg.csumoffset = offsetof(struct tcphdr, check) / 2;
755 	arg.flags = (sk && inet_sk_transparent(sk)) ? IP_REPLY_ARG_NOSRCCHECK : 0;
756 
757 	/* When socket is gone, all binding information is lost.
758 	 * routing might fail in this case. No choice here, if we choose to force
759 	 * input interface, we will misroute in case of asymmetric route.
760 	 */
761 	if (sk) {
762 		arg.bound_dev_if = sk->sk_bound_dev_if;
763 		if (sk_fullsock(sk))
764 			trace_tcp_send_reset(sk, skb);
765 	}
766 
767 	BUILD_BUG_ON(offsetof(struct sock, sk_bound_dev_if) !=
768 		     offsetof(struct inet_timewait_sock, tw_bound_dev_if));
769 
770 	arg.tos = ip_hdr(skb)->tos;
771 	arg.uid = sock_net_uid(net, sk && sk_fullsock(sk) ? sk : NULL);
772 	local_bh_disable();
773 	ctl_sk = this_cpu_read(*net->ipv4.tcp_sk);
774 	if (sk) {
775 		ctl_sk->sk_mark = (sk->sk_state == TCP_TIME_WAIT) ?
776 				   inet_twsk(sk)->tw_mark : sk->sk_mark;
777 		ctl_sk->sk_priority = (sk->sk_state == TCP_TIME_WAIT) ?
778 				   inet_twsk(sk)->tw_priority : sk->sk_priority;
779 		transmit_time = tcp_transmit_time(sk);
780 	}
781 	ip_send_unicast_reply(ctl_sk,
782 			      skb, &TCP_SKB_CB(skb)->header.h4.opt,
783 			      ip_hdr(skb)->saddr, ip_hdr(skb)->daddr,
784 			      &arg, arg.iov[0].iov_len,
785 			      transmit_time);
786 
787 	ctl_sk->sk_mark = 0;
788 	__TCP_INC_STATS(net, TCP_MIB_OUTSEGS);
789 	__TCP_INC_STATS(net, TCP_MIB_OUTRSTS);
790 	local_bh_enable();
791 
792 #ifdef CONFIG_TCP_MD5SIG
793 out:
794 	rcu_read_unlock();
795 #endif
796 }
797 
798 /* The code following below sending ACKs in SYN-RECV and TIME-WAIT states
799    outside socket context is ugly, certainly. What can I do?
800  */
801 
802 static void tcp_v4_send_ack(const struct sock *sk,
803 			    struct sk_buff *skb, u32 seq, u32 ack,
804 			    u32 win, u32 tsval, u32 tsecr, int oif,
805 			    struct tcp_md5sig_key *key,
806 			    int reply_flags, u8 tos)
807 {
808 	const struct tcphdr *th = tcp_hdr(skb);
809 	struct {
810 		struct tcphdr th;
811 		__be32 opt[(TCPOLEN_TSTAMP_ALIGNED >> 2)
812 #ifdef CONFIG_TCP_MD5SIG
813 			   + (TCPOLEN_MD5SIG_ALIGNED >> 2)
814 #endif
815 			];
816 	} rep;
817 	struct net *net = sock_net(sk);
818 	struct ip_reply_arg arg;
819 	struct sock *ctl_sk;
820 	u64 transmit_time;
821 
822 	memset(&rep.th, 0, sizeof(struct tcphdr));
823 	memset(&arg, 0, sizeof(arg));
824 
825 	arg.iov[0].iov_base = (unsigned char *)&rep;
826 	arg.iov[0].iov_len  = sizeof(rep.th);
827 	if (tsecr) {
828 		rep.opt[0] = htonl((TCPOPT_NOP << 24) | (TCPOPT_NOP << 16) |
829 				   (TCPOPT_TIMESTAMP << 8) |
830 				   TCPOLEN_TIMESTAMP);
831 		rep.opt[1] = htonl(tsval);
832 		rep.opt[2] = htonl(tsecr);
833 		arg.iov[0].iov_len += TCPOLEN_TSTAMP_ALIGNED;
834 	}
835 
836 	/* Swap the send and the receive. */
837 	rep.th.dest    = th->source;
838 	rep.th.source  = th->dest;
839 	rep.th.doff    = arg.iov[0].iov_len / 4;
840 	rep.th.seq     = htonl(seq);
841 	rep.th.ack_seq = htonl(ack);
842 	rep.th.ack     = 1;
843 	rep.th.window  = htons(win);
844 
845 #ifdef CONFIG_TCP_MD5SIG
846 	if (key) {
847 		int offset = (tsecr) ? 3 : 0;
848 
849 		rep.opt[offset++] = htonl((TCPOPT_NOP << 24) |
850 					  (TCPOPT_NOP << 16) |
851 					  (TCPOPT_MD5SIG << 8) |
852 					  TCPOLEN_MD5SIG);
853 		arg.iov[0].iov_len += TCPOLEN_MD5SIG_ALIGNED;
854 		rep.th.doff = arg.iov[0].iov_len/4;
855 
856 		tcp_v4_md5_hash_hdr((__u8 *) &rep.opt[offset],
857 				    key, ip_hdr(skb)->saddr,
858 				    ip_hdr(skb)->daddr, &rep.th);
859 	}
860 #endif
861 	arg.flags = reply_flags;
862 	arg.csum = csum_tcpudp_nofold(ip_hdr(skb)->daddr,
863 				      ip_hdr(skb)->saddr, /* XXX */
864 				      arg.iov[0].iov_len, IPPROTO_TCP, 0);
865 	arg.csumoffset = offsetof(struct tcphdr, check) / 2;
866 	if (oif)
867 		arg.bound_dev_if = oif;
868 	arg.tos = tos;
869 	arg.uid = sock_net_uid(net, sk_fullsock(sk) ? sk : NULL);
870 	local_bh_disable();
871 	ctl_sk = this_cpu_read(*net->ipv4.tcp_sk);
872 	ctl_sk->sk_mark = (sk->sk_state == TCP_TIME_WAIT) ?
873 			   inet_twsk(sk)->tw_mark : sk->sk_mark;
874 	ctl_sk->sk_priority = (sk->sk_state == TCP_TIME_WAIT) ?
875 			   inet_twsk(sk)->tw_priority : sk->sk_priority;
876 	transmit_time = tcp_transmit_time(sk);
877 	ip_send_unicast_reply(ctl_sk,
878 			      skb, &TCP_SKB_CB(skb)->header.h4.opt,
879 			      ip_hdr(skb)->saddr, ip_hdr(skb)->daddr,
880 			      &arg, arg.iov[0].iov_len,
881 			      transmit_time);
882 
883 	ctl_sk->sk_mark = 0;
884 	__TCP_INC_STATS(net, TCP_MIB_OUTSEGS);
885 	local_bh_enable();
886 }
887 
888 static void tcp_v4_timewait_ack(struct sock *sk, struct sk_buff *skb)
889 {
890 	struct inet_timewait_sock *tw = inet_twsk(sk);
891 	struct tcp_timewait_sock *tcptw = tcp_twsk(sk);
892 
893 	tcp_v4_send_ack(sk, skb,
894 			tcptw->tw_snd_nxt, tcptw->tw_rcv_nxt,
895 			tcptw->tw_rcv_wnd >> tw->tw_rcv_wscale,
896 			tcp_time_stamp_raw() + tcptw->tw_ts_offset,
897 			tcptw->tw_ts_recent,
898 			tw->tw_bound_dev_if,
899 			tcp_twsk_md5_key(tcptw),
900 			tw->tw_transparent ? IP_REPLY_ARG_NOSRCCHECK : 0,
901 			tw->tw_tos
902 			);
903 
904 	inet_twsk_put(tw);
905 }
906 
907 static void tcp_v4_reqsk_send_ack(const struct sock *sk, struct sk_buff *skb,
908 				  struct request_sock *req)
909 {
910 	/* sk->sk_state == TCP_LISTEN -> for regular TCP_SYN_RECV
911 	 * sk->sk_state == TCP_SYN_RECV -> for Fast Open.
912 	 */
913 	u32 seq = (sk->sk_state == TCP_LISTEN) ? tcp_rsk(req)->snt_isn + 1 :
914 					     tcp_sk(sk)->snd_nxt;
915 
916 	/* RFC 7323 2.3
917 	 * The window field (SEG.WND) of every outgoing segment, with the
918 	 * exception of <SYN> segments, MUST be right-shifted by
919 	 * Rcv.Wind.Shift bits:
920 	 */
921 	tcp_v4_send_ack(sk, skb, seq,
922 			tcp_rsk(req)->rcv_nxt,
923 			req->rsk_rcv_wnd >> inet_rsk(req)->rcv_wscale,
924 			tcp_time_stamp_raw() + tcp_rsk(req)->ts_off,
925 			req->ts_recent,
926 			0,
927 			tcp_md5_do_lookup(sk, (union tcp_md5_addr *)&ip_hdr(skb)->saddr,
928 					  AF_INET),
929 			inet_rsk(req)->no_srccheck ? IP_REPLY_ARG_NOSRCCHECK : 0,
930 			ip_hdr(skb)->tos);
931 }
932 
933 /*
934  *	Send a SYN-ACK after having received a SYN.
935  *	This still operates on a request_sock only, not on a big
936  *	socket.
937  */
938 static int tcp_v4_send_synack(const struct sock *sk, struct dst_entry *dst,
939 			      struct flowi *fl,
940 			      struct request_sock *req,
941 			      struct tcp_fastopen_cookie *foc,
942 			      enum tcp_synack_type synack_type)
943 {
944 	const struct inet_request_sock *ireq = inet_rsk(req);
945 	struct flowi4 fl4;
946 	int err = -1;
947 	struct sk_buff *skb;
948 
949 	/* First, grab a route. */
950 	if (!dst && (dst = inet_csk_route_req(sk, &fl4, req)) == NULL)
951 		return -1;
952 
953 	skb = tcp_make_synack(sk, dst, req, foc, synack_type);
954 
955 	if (skb) {
956 		__tcp_v4_send_check(skb, ireq->ir_loc_addr, ireq->ir_rmt_addr);
957 
958 		rcu_read_lock();
959 		err = ip_build_and_send_pkt(skb, sk, ireq->ir_loc_addr,
960 					    ireq->ir_rmt_addr,
961 					    rcu_dereference(ireq->ireq_opt));
962 		rcu_read_unlock();
963 		err = net_xmit_eval(err);
964 	}
965 
966 	return err;
967 }
968 
969 /*
970  *	IPv4 request_sock destructor.
971  */
972 static void tcp_v4_reqsk_destructor(struct request_sock *req)
973 {
974 	kfree(rcu_dereference_protected(inet_rsk(req)->ireq_opt, 1));
975 }
976 
977 #ifdef CONFIG_TCP_MD5SIG
978 /*
979  * RFC2385 MD5 checksumming requires a mapping of
980  * IP address->MD5 Key.
981  * We need to maintain these in the sk structure.
982  */
983 
984 DEFINE_STATIC_KEY_FALSE(tcp_md5_needed);
985 EXPORT_SYMBOL(tcp_md5_needed);
986 
987 /* Find the Key structure for an address.  */
988 struct tcp_md5sig_key *__tcp_md5_do_lookup(const struct sock *sk,
989 					   const union tcp_md5_addr *addr,
990 					   int family)
991 {
992 	const struct tcp_sock *tp = tcp_sk(sk);
993 	struct tcp_md5sig_key *key;
994 	const struct tcp_md5sig_info *md5sig;
995 	__be32 mask;
996 	struct tcp_md5sig_key *best_match = NULL;
997 	bool match;
998 
999 	/* caller either holds rcu_read_lock() or socket lock */
1000 	md5sig = rcu_dereference_check(tp->md5sig_info,
1001 				       lockdep_sock_is_held(sk));
1002 	if (!md5sig)
1003 		return NULL;
1004 
1005 	hlist_for_each_entry_rcu(key, &md5sig->head, node) {
1006 		if (key->family != family)
1007 			continue;
1008 
1009 		if (family == AF_INET) {
1010 			mask = inet_make_mask(key->prefixlen);
1011 			match = (key->addr.a4.s_addr & mask) ==
1012 				(addr->a4.s_addr & mask);
1013 #if IS_ENABLED(CONFIG_IPV6)
1014 		} else if (family == AF_INET6) {
1015 			match = ipv6_prefix_equal(&key->addr.a6, &addr->a6,
1016 						  key->prefixlen);
1017 #endif
1018 		} else {
1019 			match = false;
1020 		}
1021 
1022 		if (match && (!best_match ||
1023 			      key->prefixlen > best_match->prefixlen))
1024 			best_match = key;
1025 	}
1026 	return best_match;
1027 }
1028 EXPORT_SYMBOL(__tcp_md5_do_lookup);
1029 
1030 static struct tcp_md5sig_key *tcp_md5_do_lookup_exact(const struct sock *sk,
1031 						      const union tcp_md5_addr *addr,
1032 						      int family, u8 prefixlen)
1033 {
1034 	const struct tcp_sock *tp = tcp_sk(sk);
1035 	struct tcp_md5sig_key *key;
1036 	unsigned int size = sizeof(struct in_addr);
1037 	const struct tcp_md5sig_info *md5sig;
1038 
1039 	/* caller either holds rcu_read_lock() or socket lock */
1040 	md5sig = rcu_dereference_check(tp->md5sig_info,
1041 				       lockdep_sock_is_held(sk));
1042 	if (!md5sig)
1043 		return NULL;
1044 #if IS_ENABLED(CONFIG_IPV6)
1045 	if (family == AF_INET6)
1046 		size = sizeof(struct in6_addr);
1047 #endif
1048 	hlist_for_each_entry_rcu(key, &md5sig->head, node) {
1049 		if (key->family != family)
1050 			continue;
1051 		if (!memcmp(&key->addr, addr, size) &&
1052 		    key->prefixlen == prefixlen)
1053 			return key;
1054 	}
1055 	return NULL;
1056 }
1057 
1058 struct tcp_md5sig_key *tcp_v4_md5_lookup(const struct sock *sk,
1059 					 const struct sock *addr_sk)
1060 {
1061 	const union tcp_md5_addr *addr;
1062 
1063 	addr = (const union tcp_md5_addr *)&addr_sk->sk_daddr;
1064 	return tcp_md5_do_lookup(sk, addr, AF_INET);
1065 }
1066 EXPORT_SYMBOL(tcp_v4_md5_lookup);
1067 
1068 /* This can be called on a newly created socket, from other files */
1069 int tcp_md5_do_add(struct sock *sk, const union tcp_md5_addr *addr,
1070 		   int family, u8 prefixlen, const u8 *newkey, u8 newkeylen,
1071 		   gfp_t gfp)
1072 {
1073 	/* Add Key to the list */
1074 	struct tcp_md5sig_key *key;
1075 	struct tcp_sock *tp = tcp_sk(sk);
1076 	struct tcp_md5sig_info *md5sig;
1077 
1078 	key = tcp_md5_do_lookup_exact(sk, addr, family, prefixlen);
1079 	if (key) {
1080 		/* Pre-existing entry - just update that one. */
1081 		memcpy(key->key, newkey, newkeylen);
1082 		key->keylen = newkeylen;
1083 		return 0;
1084 	}
1085 
1086 	md5sig = rcu_dereference_protected(tp->md5sig_info,
1087 					   lockdep_sock_is_held(sk));
1088 	if (!md5sig) {
1089 		md5sig = kmalloc(sizeof(*md5sig), gfp);
1090 		if (!md5sig)
1091 			return -ENOMEM;
1092 
1093 		sk_nocaps_add(sk, NETIF_F_GSO_MASK);
1094 		INIT_HLIST_HEAD(&md5sig->head);
1095 		rcu_assign_pointer(tp->md5sig_info, md5sig);
1096 	}
1097 
1098 	key = sock_kmalloc(sk, sizeof(*key), gfp);
1099 	if (!key)
1100 		return -ENOMEM;
1101 	if (!tcp_alloc_md5sig_pool()) {
1102 		sock_kfree_s(sk, key, sizeof(*key));
1103 		return -ENOMEM;
1104 	}
1105 
1106 	memcpy(key->key, newkey, newkeylen);
1107 	key->keylen = newkeylen;
1108 	key->family = family;
1109 	key->prefixlen = prefixlen;
1110 	memcpy(&key->addr, addr,
1111 	       (family == AF_INET6) ? sizeof(struct in6_addr) :
1112 				      sizeof(struct in_addr));
1113 	hlist_add_head_rcu(&key->node, &md5sig->head);
1114 	return 0;
1115 }
1116 EXPORT_SYMBOL(tcp_md5_do_add);
1117 
1118 int tcp_md5_do_del(struct sock *sk, const union tcp_md5_addr *addr, int family,
1119 		   u8 prefixlen)
1120 {
1121 	struct tcp_md5sig_key *key;
1122 
1123 	key = tcp_md5_do_lookup_exact(sk, addr, family, prefixlen);
1124 	if (!key)
1125 		return -ENOENT;
1126 	hlist_del_rcu(&key->node);
1127 	atomic_sub(sizeof(*key), &sk->sk_omem_alloc);
1128 	kfree_rcu(key, rcu);
1129 	return 0;
1130 }
1131 EXPORT_SYMBOL(tcp_md5_do_del);
1132 
1133 static void tcp_clear_md5_list(struct sock *sk)
1134 {
1135 	struct tcp_sock *tp = tcp_sk(sk);
1136 	struct tcp_md5sig_key *key;
1137 	struct hlist_node *n;
1138 	struct tcp_md5sig_info *md5sig;
1139 
1140 	md5sig = rcu_dereference_protected(tp->md5sig_info, 1);
1141 
1142 	hlist_for_each_entry_safe(key, n, &md5sig->head, node) {
1143 		hlist_del_rcu(&key->node);
1144 		atomic_sub(sizeof(*key), &sk->sk_omem_alloc);
1145 		kfree_rcu(key, rcu);
1146 	}
1147 }
1148 
1149 static int tcp_v4_parse_md5_keys(struct sock *sk, int optname,
1150 				 char __user *optval, int optlen)
1151 {
1152 	struct tcp_md5sig cmd;
1153 	struct sockaddr_in *sin = (struct sockaddr_in *)&cmd.tcpm_addr;
1154 	u8 prefixlen = 32;
1155 
1156 	if (optlen < sizeof(cmd))
1157 		return -EINVAL;
1158 
1159 	if (copy_from_user(&cmd, optval, sizeof(cmd)))
1160 		return -EFAULT;
1161 
1162 	if (sin->sin_family != AF_INET)
1163 		return -EINVAL;
1164 
1165 	if (optname == TCP_MD5SIG_EXT &&
1166 	    cmd.tcpm_flags & TCP_MD5SIG_FLAG_PREFIX) {
1167 		prefixlen = cmd.tcpm_prefixlen;
1168 		if (prefixlen > 32)
1169 			return -EINVAL;
1170 	}
1171 
1172 	if (!cmd.tcpm_keylen)
1173 		return tcp_md5_do_del(sk, (union tcp_md5_addr *)&sin->sin_addr.s_addr,
1174 				      AF_INET, prefixlen);
1175 
1176 	if (cmd.tcpm_keylen > TCP_MD5SIG_MAXKEYLEN)
1177 		return -EINVAL;
1178 
1179 	return tcp_md5_do_add(sk, (union tcp_md5_addr *)&sin->sin_addr.s_addr,
1180 			      AF_INET, prefixlen, cmd.tcpm_key, cmd.tcpm_keylen,
1181 			      GFP_KERNEL);
1182 }
1183 
1184 static int tcp_v4_md5_hash_headers(struct tcp_md5sig_pool *hp,
1185 				   __be32 daddr, __be32 saddr,
1186 				   const struct tcphdr *th, int nbytes)
1187 {
1188 	struct tcp4_pseudohdr *bp;
1189 	struct scatterlist sg;
1190 	struct tcphdr *_th;
1191 
1192 	bp = hp->scratch;
1193 	bp->saddr = saddr;
1194 	bp->daddr = daddr;
1195 	bp->pad = 0;
1196 	bp->protocol = IPPROTO_TCP;
1197 	bp->len = cpu_to_be16(nbytes);
1198 
1199 	_th = (struct tcphdr *)(bp + 1);
1200 	memcpy(_th, th, sizeof(*th));
1201 	_th->check = 0;
1202 
1203 	sg_init_one(&sg, bp, sizeof(*bp) + sizeof(*th));
1204 	ahash_request_set_crypt(hp->md5_req, &sg, NULL,
1205 				sizeof(*bp) + sizeof(*th));
1206 	return crypto_ahash_update(hp->md5_req);
1207 }
1208 
1209 static int tcp_v4_md5_hash_hdr(char *md5_hash, const struct tcp_md5sig_key *key,
1210 			       __be32 daddr, __be32 saddr, const struct tcphdr *th)
1211 {
1212 	struct tcp_md5sig_pool *hp;
1213 	struct ahash_request *req;
1214 
1215 	hp = tcp_get_md5sig_pool();
1216 	if (!hp)
1217 		goto clear_hash_noput;
1218 	req = hp->md5_req;
1219 
1220 	if (crypto_ahash_init(req))
1221 		goto clear_hash;
1222 	if (tcp_v4_md5_hash_headers(hp, daddr, saddr, th, th->doff << 2))
1223 		goto clear_hash;
1224 	if (tcp_md5_hash_key(hp, key))
1225 		goto clear_hash;
1226 	ahash_request_set_crypt(req, NULL, md5_hash, 0);
1227 	if (crypto_ahash_final(req))
1228 		goto clear_hash;
1229 
1230 	tcp_put_md5sig_pool();
1231 	return 0;
1232 
1233 clear_hash:
1234 	tcp_put_md5sig_pool();
1235 clear_hash_noput:
1236 	memset(md5_hash, 0, 16);
1237 	return 1;
1238 }
1239 
1240 int tcp_v4_md5_hash_skb(char *md5_hash, const struct tcp_md5sig_key *key,
1241 			const struct sock *sk,
1242 			const struct sk_buff *skb)
1243 {
1244 	struct tcp_md5sig_pool *hp;
1245 	struct ahash_request *req;
1246 	const struct tcphdr *th = tcp_hdr(skb);
1247 	__be32 saddr, daddr;
1248 
1249 	if (sk) { /* valid for establish/request sockets */
1250 		saddr = sk->sk_rcv_saddr;
1251 		daddr = sk->sk_daddr;
1252 	} else {
1253 		const struct iphdr *iph = ip_hdr(skb);
1254 		saddr = iph->saddr;
1255 		daddr = iph->daddr;
1256 	}
1257 
1258 	hp = tcp_get_md5sig_pool();
1259 	if (!hp)
1260 		goto clear_hash_noput;
1261 	req = hp->md5_req;
1262 
1263 	if (crypto_ahash_init(req))
1264 		goto clear_hash;
1265 
1266 	if (tcp_v4_md5_hash_headers(hp, daddr, saddr, th, skb->len))
1267 		goto clear_hash;
1268 	if (tcp_md5_hash_skb_data(hp, skb, th->doff << 2))
1269 		goto clear_hash;
1270 	if (tcp_md5_hash_key(hp, key))
1271 		goto clear_hash;
1272 	ahash_request_set_crypt(req, NULL, md5_hash, 0);
1273 	if (crypto_ahash_final(req))
1274 		goto clear_hash;
1275 
1276 	tcp_put_md5sig_pool();
1277 	return 0;
1278 
1279 clear_hash:
1280 	tcp_put_md5sig_pool();
1281 clear_hash_noput:
1282 	memset(md5_hash, 0, 16);
1283 	return 1;
1284 }
1285 EXPORT_SYMBOL(tcp_v4_md5_hash_skb);
1286 
1287 #endif
1288 
1289 /* Called with rcu_read_lock() */
1290 static bool tcp_v4_inbound_md5_hash(const struct sock *sk,
1291 				    const struct sk_buff *skb)
1292 {
1293 #ifdef CONFIG_TCP_MD5SIG
1294 	/*
1295 	 * This gets called for each TCP segment that arrives
1296 	 * so we want to be efficient.
1297 	 * We have 3 drop cases:
1298 	 * o No MD5 hash and one expected.
1299 	 * o MD5 hash and we're not expecting one.
1300 	 * o MD5 hash and its wrong.
1301 	 */
1302 	const __u8 *hash_location = NULL;
1303 	struct tcp_md5sig_key *hash_expected;
1304 	const struct iphdr *iph = ip_hdr(skb);
1305 	const struct tcphdr *th = tcp_hdr(skb);
1306 	int genhash;
1307 	unsigned char newhash[16];
1308 
1309 	hash_expected = tcp_md5_do_lookup(sk, (union tcp_md5_addr *)&iph->saddr,
1310 					  AF_INET);
1311 	hash_location = tcp_parse_md5sig_option(th);
1312 
1313 	/* We've parsed the options - do we have a hash? */
1314 	if (!hash_expected && !hash_location)
1315 		return false;
1316 
1317 	if (hash_expected && !hash_location) {
1318 		NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPMD5NOTFOUND);
1319 		return true;
1320 	}
1321 
1322 	if (!hash_expected && hash_location) {
1323 		NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPMD5UNEXPECTED);
1324 		return true;
1325 	}
1326 
1327 	/* Okay, so this is hash_expected and hash_location -
1328 	 * so we need to calculate the checksum.
1329 	 */
1330 	genhash = tcp_v4_md5_hash_skb(newhash,
1331 				      hash_expected,
1332 				      NULL, skb);
1333 
1334 	if (genhash || memcmp(hash_location, newhash, 16) != 0) {
1335 		NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPMD5FAILURE);
1336 		net_info_ratelimited("MD5 Hash failed for (%pI4, %d)->(%pI4, %d)%s\n",
1337 				     &iph->saddr, ntohs(th->source),
1338 				     &iph->daddr, ntohs(th->dest),
1339 				     genhash ? " tcp_v4_calc_md5_hash failed"
1340 				     : "");
1341 		return true;
1342 	}
1343 	return false;
1344 #endif
1345 	return false;
1346 }
1347 
1348 static void tcp_v4_init_req(struct request_sock *req,
1349 			    const struct sock *sk_listener,
1350 			    struct sk_buff *skb)
1351 {
1352 	struct inet_request_sock *ireq = inet_rsk(req);
1353 	struct net *net = sock_net(sk_listener);
1354 
1355 	sk_rcv_saddr_set(req_to_sk(req), ip_hdr(skb)->daddr);
1356 	sk_daddr_set(req_to_sk(req), ip_hdr(skb)->saddr);
1357 	RCU_INIT_POINTER(ireq->ireq_opt, tcp_v4_save_options(net, skb));
1358 }
1359 
1360 static struct dst_entry *tcp_v4_route_req(const struct sock *sk,
1361 					  struct flowi *fl,
1362 					  const struct request_sock *req)
1363 {
1364 	return inet_csk_route_req(sk, &fl->u.ip4, req);
1365 }
1366 
1367 struct request_sock_ops tcp_request_sock_ops __read_mostly = {
1368 	.family		=	PF_INET,
1369 	.obj_size	=	sizeof(struct tcp_request_sock),
1370 	.rtx_syn_ack	=	tcp_rtx_synack,
1371 	.send_ack	=	tcp_v4_reqsk_send_ack,
1372 	.destructor	=	tcp_v4_reqsk_destructor,
1373 	.send_reset	=	tcp_v4_send_reset,
1374 	.syn_ack_timeout =	tcp_syn_ack_timeout,
1375 };
1376 
1377 static const struct tcp_request_sock_ops tcp_request_sock_ipv4_ops = {
1378 	.mss_clamp	=	TCP_MSS_DEFAULT,
1379 #ifdef CONFIG_TCP_MD5SIG
1380 	.req_md5_lookup	=	tcp_v4_md5_lookup,
1381 	.calc_md5_hash	=	tcp_v4_md5_hash_skb,
1382 #endif
1383 	.init_req	=	tcp_v4_init_req,
1384 #ifdef CONFIG_SYN_COOKIES
1385 	.cookie_init_seq =	cookie_v4_init_sequence,
1386 #endif
1387 	.route_req	=	tcp_v4_route_req,
1388 	.init_seq	=	tcp_v4_init_seq,
1389 	.init_ts_off	=	tcp_v4_init_ts_off,
1390 	.send_synack	=	tcp_v4_send_synack,
1391 };
1392 
1393 int tcp_v4_conn_request(struct sock *sk, struct sk_buff *skb)
1394 {
1395 	/* Never answer to SYNs send to broadcast or multicast */
1396 	if (skb_rtable(skb)->rt_flags & (RTCF_BROADCAST | RTCF_MULTICAST))
1397 		goto drop;
1398 
1399 	return tcp_conn_request(&tcp_request_sock_ops,
1400 				&tcp_request_sock_ipv4_ops, sk, skb);
1401 
1402 drop:
1403 	tcp_listendrop(sk);
1404 	return 0;
1405 }
1406 EXPORT_SYMBOL(tcp_v4_conn_request);
1407 
1408 
1409 /*
1410  * The three way handshake has completed - we got a valid synack -
1411  * now create the new socket.
1412  */
1413 struct sock *tcp_v4_syn_recv_sock(const struct sock *sk, struct sk_buff *skb,
1414 				  struct request_sock *req,
1415 				  struct dst_entry *dst,
1416 				  struct request_sock *req_unhash,
1417 				  bool *own_req)
1418 {
1419 	struct inet_request_sock *ireq;
1420 	struct inet_sock *newinet;
1421 	struct tcp_sock *newtp;
1422 	struct sock *newsk;
1423 #ifdef CONFIG_TCP_MD5SIG
1424 	struct tcp_md5sig_key *key;
1425 #endif
1426 	struct ip_options_rcu *inet_opt;
1427 
1428 	if (sk_acceptq_is_full(sk))
1429 		goto exit_overflow;
1430 
1431 	newsk = tcp_create_openreq_child(sk, req, skb);
1432 	if (!newsk)
1433 		goto exit_nonewsk;
1434 
1435 	newsk->sk_gso_type = SKB_GSO_TCPV4;
1436 	inet_sk_rx_dst_set(newsk, skb);
1437 
1438 	newtp		      = tcp_sk(newsk);
1439 	newinet		      = inet_sk(newsk);
1440 	ireq		      = inet_rsk(req);
1441 	sk_daddr_set(newsk, ireq->ir_rmt_addr);
1442 	sk_rcv_saddr_set(newsk, ireq->ir_loc_addr);
1443 	newsk->sk_bound_dev_if = ireq->ir_iif;
1444 	newinet->inet_saddr   = ireq->ir_loc_addr;
1445 	inet_opt	      = rcu_dereference(ireq->ireq_opt);
1446 	RCU_INIT_POINTER(newinet->inet_opt, inet_opt);
1447 	newinet->mc_index     = inet_iif(skb);
1448 	newinet->mc_ttl	      = ip_hdr(skb)->ttl;
1449 	newinet->rcv_tos      = ip_hdr(skb)->tos;
1450 	inet_csk(newsk)->icsk_ext_hdr_len = 0;
1451 	if (inet_opt)
1452 		inet_csk(newsk)->icsk_ext_hdr_len = inet_opt->opt.optlen;
1453 	newinet->inet_id = prandom_u32();
1454 
1455 	if (!dst) {
1456 		dst = inet_csk_route_child_sock(sk, newsk, req);
1457 		if (!dst)
1458 			goto put_and_exit;
1459 	} else {
1460 		/* syncookie case : see end of cookie_v4_check() */
1461 	}
1462 	sk_setup_caps(newsk, dst);
1463 
1464 	tcp_ca_openreq_child(newsk, dst);
1465 
1466 	tcp_sync_mss(newsk, dst_mtu(dst));
1467 	newtp->advmss = tcp_mss_clamp(tcp_sk(sk), dst_metric_advmss(dst));
1468 
1469 	tcp_initialize_rcv_mss(newsk);
1470 
1471 #ifdef CONFIG_TCP_MD5SIG
1472 	/* Copy over the MD5 key from the original socket */
1473 	key = tcp_md5_do_lookup(sk, (union tcp_md5_addr *)&newinet->inet_daddr,
1474 				AF_INET);
1475 	if (key) {
1476 		/*
1477 		 * We're using one, so create a matching key
1478 		 * on the newsk structure. If we fail to get
1479 		 * memory, then we end up not copying the key
1480 		 * across. Shucks.
1481 		 */
1482 		tcp_md5_do_add(newsk, (union tcp_md5_addr *)&newinet->inet_daddr,
1483 			       AF_INET, 32, key->key, key->keylen, GFP_ATOMIC);
1484 		sk_nocaps_add(newsk, NETIF_F_GSO_MASK);
1485 	}
1486 #endif
1487 
1488 	if (__inet_inherit_port(sk, newsk) < 0)
1489 		goto put_and_exit;
1490 	*own_req = inet_ehash_nolisten(newsk, req_to_sk(req_unhash));
1491 	if (likely(*own_req)) {
1492 		tcp_move_syn(newtp, req);
1493 		ireq->ireq_opt = NULL;
1494 	} else {
1495 		newinet->inet_opt = NULL;
1496 	}
1497 	return newsk;
1498 
1499 exit_overflow:
1500 	NET_INC_STATS(sock_net(sk), LINUX_MIB_LISTENOVERFLOWS);
1501 exit_nonewsk:
1502 	dst_release(dst);
1503 exit:
1504 	tcp_listendrop(sk);
1505 	return NULL;
1506 put_and_exit:
1507 	newinet->inet_opt = NULL;
1508 	inet_csk_prepare_forced_close(newsk);
1509 	tcp_done(newsk);
1510 	goto exit;
1511 }
1512 EXPORT_SYMBOL(tcp_v4_syn_recv_sock);
1513 
1514 static struct sock *tcp_v4_cookie_check(struct sock *sk, struct sk_buff *skb)
1515 {
1516 #ifdef CONFIG_SYN_COOKIES
1517 	const struct tcphdr *th = tcp_hdr(skb);
1518 
1519 	if (!th->syn)
1520 		sk = cookie_v4_check(sk, skb);
1521 #endif
1522 	return sk;
1523 }
1524 
1525 u16 tcp_v4_get_syncookie(struct sock *sk, struct iphdr *iph,
1526 			 struct tcphdr *th, u32 *cookie)
1527 {
1528 	u16 mss = 0;
1529 #ifdef CONFIG_SYN_COOKIES
1530 	mss = tcp_get_syncookie_mss(&tcp_request_sock_ops,
1531 				    &tcp_request_sock_ipv4_ops, sk, th);
1532 	if (mss) {
1533 		*cookie = __cookie_v4_init_sequence(iph, th, &mss);
1534 		tcp_synq_overflow(sk);
1535 	}
1536 #endif
1537 	return mss;
1538 }
1539 
1540 /* The socket must have it's spinlock held when we get
1541  * here, unless it is a TCP_LISTEN socket.
1542  *
1543  * We have a potential double-lock case here, so even when
1544  * doing backlog processing we use the BH locking scheme.
1545  * This is because we cannot sleep with the original spinlock
1546  * held.
1547  */
1548 int tcp_v4_do_rcv(struct sock *sk, struct sk_buff *skb)
1549 {
1550 	struct sock *rsk;
1551 
1552 	if (sk->sk_state == TCP_ESTABLISHED) { /* Fast path */
1553 		struct dst_entry *dst = sk->sk_rx_dst;
1554 
1555 		sock_rps_save_rxhash(sk, skb);
1556 		sk_mark_napi_id(sk, skb);
1557 		if (dst) {
1558 			if (inet_sk(sk)->rx_dst_ifindex != skb->skb_iif ||
1559 			    !dst->ops->check(dst, 0)) {
1560 				dst_release(dst);
1561 				sk->sk_rx_dst = NULL;
1562 			}
1563 		}
1564 		tcp_rcv_established(sk, skb);
1565 		return 0;
1566 	}
1567 
1568 	if (tcp_checksum_complete(skb))
1569 		goto csum_err;
1570 
1571 	if (sk->sk_state == TCP_LISTEN) {
1572 		struct sock *nsk = tcp_v4_cookie_check(sk, skb);
1573 
1574 		if (!nsk)
1575 			goto discard;
1576 		if (nsk != sk) {
1577 			if (tcp_child_process(sk, nsk, skb)) {
1578 				rsk = nsk;
1579 				goto reset;
1580 			}
1581 			return 0;
1582 		}
1583 	} else
1584 		sock_rps_save_rxhash(sk, skb);
1585 
1586 	if (tcp_rcv_state_process(sk, skb)) {
1587 		rsk = sk;
1588 		goto reset;
1589 	}
1590 	return 0;
1591 
1592 reset:
1593 	tcp_v4_send_reset(rsk, skb);
1594 discard:
1595 	kfree_skb(skb);
1596 	/* Be careful here. If this function gets more complicated and
1597 	 * gcc suffers from register pressure on the x86, sk (in %ebx)
1598 	 * might be destroyed here. This current version compiles correctly,
1599 	 * but you have been warned.
1600 	 */
1601 	return 0;
1602 
1603 csum_err:
1604 	TCP_INC_STATS(sock_net(sk), TCP_MIB_CSUMERRORS);
1605 	TCP_INC_STATS(sock_net(sk), TCP_MIB_INERRS);
1606 	goto discard;
1607 }
1608 EXPORT_SYMBOL(tcp_v4_do_rcv);
1609 
1610 int tcp_v4_early_demux(struct sk_buff *skb)
1611 {
1612 	const struct iphdr *iph;
1613 	const struct tcphdr *th;
1614 	struct sock *sk;
1615 
1616 	if (skb->pkt_type != PACKET_HOST)
1617 		return 0;
1618 
1619 	if (!pskb_may_pull(skb, skb_transport_offset(skb) + sizeof(struct tcphdr)))
1620 		return 0;
1621 
1622 	iph = ip_hdr(skb);
1623 	th = tcp_hdr(skb);
1624 
1625 	if (th->doff < sizeof(struct tcphdr) / 4)
1626 		return 0;
1627 
1628 	sk = __inet_lookup_established(dev_net(skb->dev), &tcp_hashinfo,
1629 				       iph->saddr, th->source,
1630 				       iph->daddr, ntohs(th->dest),
1631 				       skb->skb_iif, inet_sdif(skb));
1632 	if (sk) {
1633 		skb->sk = sk;
1634 		skb->destructor = sock_edemux;
1635 		if (sk_fullsock(sk)) {
1636 			struct dst_entry *dst = READ_ONCE(sk->sk_rx_dst);
1637 
1638 			if (dst)
1639 				dst = dst_check(dst, 0);
1640 			if (dst &&
1641 			    inet_sk(sk)->rx_dst_ifindex == skb->skb_iif)
1642 				skb_dst_set_noref(skb, dst);
1643 		}
1644 	}
1645 	return 0;
1646 }
1647 
1648 bool tcp_add_backlog(struct sock *sk, struct sk_buff *skb)
1649 {
1650 	u32 limit = READ_ONCE(sk->sk_rcvbuf) + READ_ONCE(sk->sk_sndbuf);
1651 	struct skb_shared_info *shinfo;
1652 	const struct tcphdr *th;
1653 	struct tcphdr *thtail;
1654 	struct sk_buff *tail;
1655 	unsigned int hdrlen;
1656 	bool fragstolen;
1657 	u32 gso_segs;
1658 	int delta;
1659 
1660 	/* In case all data was pulled from skb frags (in __pskb_pull_tail()),
1661 	 * we can fix skb->truesize to its real value to avoid future drops.
1662 	 * This is valid because skb is not yet charged to the socket.
1663 	 * It has been noticed pure SACK packets were sometimes dropped
1664 	 * (if cooked by drivers without copybreak feature).
1665 	 */
1666 	skb_condense(skb);
1667 
1668 	skb_dst_drop(skb);
1669 
1670 	if (unlikely(tcp_checksum_complete(skb))) {
1671 		bh_unlock_sock(sk);
1672 		__TCP_INC_STATS(sock_net(sk), TCP_MIB_CSUMERRORS);
1673 		__TCP_INC_STATS(sock_net(sk), TCP_MIB_INERRS);
1674 		return true;
1675 	}
1676 
1677 	/* Attempt coalescing to last skb in backlog, even if we are
1678 	 * above the limits.
1679 	 * This is okay because skb capacity is limited to MAX_SKB_FRAGS.
1680 	 */
1681 	th = (const struct tcphdr *)skb->data;
1682 	hdrlen = th->doff * 4;
1683 	shinfo = skb_shinfo(skb);
1684 
1685 	if (!shinfo->gso_size)
1686 		shinfo->gso_size = skb->len - hdrlen;
1687 
1688 	if (!shinfo->gso_segs)
1689 		shinfo->gso_segs = 1;
1690 
1691 	tail = sk->sk_backlog.tail;
1692 	if (!tail)
1693 		goto no_coalesce;
1694 	thtail = (struct tcphdr *)tail->data;
1695 
1696 	if (TCP_SKB_CB(tail)->end_seq != TCP_SKB_CB(skb)->seq ||
1697 	    TCP_SKB_CB(tail)->ip_dsfield != TCP_SKB_CB(skb)->ip_dsfield ||
1698 	    ((TCP_SKB_CB(tail)->tcp_flags |
1699 	      TCP_SKB_CB(skb)->tcp_flags) & (TCPHDR_SYN | TCPHDR_RST | TCPHDR_URG)) ||
1700 	    !((TCP_SKB_CB(tail)->tcp_flags &
1701 	      TCP_SKB_CB(skb)->tcp_flags) & TCPHDR_ACK) ||
1702 	    ((TCP_SKB_CB(tail)->tcp_flags ^
1703 	      TCP_SKB_CB(skb)->tcp_flags) & (TCPHDR_ECE | TCPHDR_CWR)) ||
1704 #ifdef CONFIG_TLS_DEVICE
1705 	    tail->decrypted != skb->decrypted ||
1706 #endif
1707 	    thtail->doff != th->doff ||
1708 	    memcmp(thtail + 1, th + 1, hdrlen - sizeof(*th)))
1709 		goto no_coalesce;
1710 
1711 	__skb_pull(skb, hdrlen);
1712 	if (skb_try_coalesce(tail, skb, &fragstolen, &delta)) {
1713 		thtail->window = th->window;
1714 
1715 		TCP_SKB_CB(tail)->end_seq = TCP_SKB_CB(skb)->end_seq;
1716 
1717 		if (after(TCP_SKB_CB(skb)->ack_seq, TCP_SKB_CB(tail)->ack_seq))
1718 			TCP_SKB_CB(tail)->ack_seq = TCP_SKB_CB(skb)->ack_seq;
1719 
1720 		/* We have to update both TCP_SKB_CB(tail)->tcp_flags and
1721 		 * thtail->fin, so that the fast path in tcp_rcv_established()
1722 		 * is not entered if we append a packet with a FIN.
1723 		 * SYN, RST, URG are not present.
1724 		 * ACK is set on both packets.
1725 		 * PSH : we do not really care in TCP stack,
1726 		 *       at least for 'GRO' packets.
1727 		 */
1728 		thtail->fin |= th->fin;
1729 		TCP_SKB_CB(tail)->tcp_flags |= TCP_SKB_CB(skb)->tcp_flags;
1730 
1731 		if (TCP_SKB_CB(skb)->has_rxtstamp) {
1732 			TCP_SKB_CB(tail)->has_rxtstamp = true;
1733 			tail->tstamp = skb->tstamp;
1734 			skb_hwtstamps(tail)->hwtstamp = skb_hwtstamps(skb)->hwtstamp;
1735 		}
1736 
1737 		/* Not as strict as GRO. We only need to carry mss max value */
1738 		skb_shinfo(tail)->gso_size = max(shinfo->gso_size,
1739 						 skb_shinfo(tail)->gso_size);
1740 
1741 		gso_segs = skb_shinfo(tail)->gso_segs + shinfo->gso_segs;
1742 		skb_shinfo(tail)->gso_segs = min_t(u32, gso_segs, 0xFFFF);
1743 
1744 		sk->sk_backlog.len += delta;
1745 		__NET_INC_STATS(sock_net(sk),
1746 				LINUX_MIB_TCPBACKLOGCOALESCE);
1747 		kfree_skb_partial(skb, fragstolen);
1748 		return false;
1749 	}
1750 	__skb_push(skb, hdrlen);
1751 
1752 no_coalesce:
1753 	/* Only socket owner can try to collapse/prune rx queues
1754 	 * to reduce memory overhead, so add a little headroom here.
1755 	 * Few sockets backlog are possibly concurrently non empty.
1756 	 */
1757 	limit += 64*1024;
1758 
1759 	if (unlikely(sk_add_backlog(sk, skb, limit))) {
1760 		bh_unlock_sock(sk);
1761 		__NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPBACKLOGDROP);
1762 		return true;
1763 	}
1764 	return false;
1765 }
1766 EXPORT_SYMBOL(tcp_add_backlog);
1767 
1768 int tcp_filter(struct sock *sk, struct sk_buff *skb)
1769 {
1770 	struct tcphdr *th = (struct tcphdr *)skb->data;
1771 
1772 	return sk_filter_trim_cap(sk, skb, th->doff * 4);
1773 }
1774 EXPORT_SYMBOL(tcp_filter);
1775 
1776 static void tcp_v4_restore_cb(struct sk_buff *skb)
1777 {
1778 	memmove(IPCB(skb), &TCP_SKB_CB(skb)->header.h4,
1779 		sizeof(struct inet_skb_parm));
1780 }
1781 
1782 static void tcp_v4_fill_cb(struct sk_buff *skb, const struct iphdr *iph,
1783 			   const struct tcphdr *th)
1784 {
1785 	/* This is tricky : We move IPCB at its correct location into TCP_SKB_CB()
1786 	 * barrier() makes sure compiler wont play fool^Waliasing games.
1787 	 */
1788 	memmove(&TCP_SKB_CB(skb)->header.h4, IPCB(skb),
1789 		sizeof(struct inet_skb_parm));
1790 	barrier();
1791 
1792 	TCP_SKB_CB(skb)->seq = ntohl(th->seq);
1793 	TCP_SKB_CB(skb)->end_seq = (TCP_SKB_CB(skb)->seq + th->syn + th->fin +
1794 				    skb->len - th->doff * 4);
1795 	TCP_SKB_CB(skb)->ack_seq = ntohl(th->ack_seq);
1796 	TCP_SKB_CB(skb)->tcp_flags = tcp_flag_byte(th);
1797 	TCP_SKB_CB(skb)->tcp_tw_isn = 0;
1798 	TCP_SKB_CB(skb)->ip_dsfield = ipv4_get_dsfield(iph);
1799 	TCP_SKB_CB(skb)->sacked	 = 0;
1800 	TCP_SKB_CB(skb)->has_rxtstamp =
1801 			skb->tstamp || skb_hwtstamps(skb)->hwtstamp;
1802 }
1803 
1804 /*
1805  *	From tcp_input.c
1806  */
1807 
1808 int tcp_v4_rcv(struct sk_buff *skb)
1809 {
1810 	struct net *net = dev_net(skb->dev);
1811 	struct sk_buff *skb_to_free;
1812 	int sdif = inet_sdif(skb);
1813 	const struct iphdr *iph;
1814 	const struct tcphdr *th;
1815 	bool refcounted;
1816 	struct sock *sk;
1817 	int ret;
1818 
1819 	if (skb->pkt_type != PACKET_HOST)
1820 		goto discard_it;
1821 
1822 	/* Count it even if it's bad */
1823 	__TCP_INC_STATS(net, TCP_MIB_INSEGS);
1824 
1825 	if (!pskb_may_pull(skb, sizeof(struct tcphdr)))
1826 		goto discard_it;
1827 
1828 	th = (const struct tcphdr *)skb->data;
1829 
1830 	if (unlikely(th->doff < sizeof(struct tcphdr) / 4))
1831 		goto bad_packet;
1832 	if (!pskb_may_pull(skb, th->doff * 4))
1833 		goto discard_it;
1834 
1835 	/* An explanation is required here, I think.
1836 	 * Packet length and doff are validated by header prediction,
1837 	 * provided case of th->doff==0 is eliminated.
1838 	 * So, we defer the checks. */
1839 
1840 	if (skb_checksum_init(skb, IPPROTO_TCP, inet_compute_pseudo))
1841 		goto csum_error;
1842 
1843 	th = (const struct tcphdr *)skb->data;
1844 	iph = ip_hdr(skb);
1845 lookup:
1846 	sk = __inet_lookup_skb(&tcp_hashinfo, skb, __tcp_hdrlen(th), th->source,
1847 			       th->dest, sdif, &refcounted);
1848 	if (!sk)
1849 		goto no_tcp_socket;
1850 
1851 process:
1852 	if (sk->sk_state == TCP_TIME_WAIT)
1853 		goto do_time_wait;
1854 
1855 	if (sk->sk_state == TCP_NEW_SYN_RECV) {
1856 		struct request_sock *req = inet_reqsk(sk);
1857 		bool req_stolen = false;
1858 		struct sock *nsk;
1859 
1860 		sk = req->rsk_listener;
1861 		if (unlikely(tcp_v4_inbound_md5_hash(sk, skb))) {
1862 			sk_drops_add(sk, skb);
1863 			reqsk_put(req);
1864 			goto discard_it;
1865 		}
1866 		if (tcp_checksum_complete(skb)) {
1867 			reqsk_put(req);
1868 			goto csum_error;
1869 		}
1870 		if (unlikely(sk->sk_state != TCP_LISTEN)) {
1871 			inet_csk_reqsk_queue_drop_and_put(sk, req);
1872 			goto lookup;
1873 		}
1874 		/* We own a reference on the listener, increase it again
1875 		 * as we might lose it too soon.
1876 		 */
1877 		sock_hold(sk);
1878 		refcounted = true;
1879 		nsk = NULL;
1880 		if (!tcp_filter(sk, skb)) {
1881 			th = (const struct tcphdr *)skb->data;
1882 			iph = ip_hdr(skb);
1883 			tcp_v4_fill_cb(skb, iph, th);
1884 			nsk = tcp_check_req(sk, skb, req, false, &req_stolen);
1885 		}
1886 		if (!nsk) {
1887 			reqsk_put(req);
1888 			if (req_stolen) {
1889 				/* Another cpu got exclusive access to req
1890 				 * and created a full blown socket.
1891 				 * Try to feed this packet to this socket
1892 				 * instead of discarding it.
1893 				 */
1894 				tcp_v4_restore_cb(skb);
1895 				sock_put(sk);
1896 				goto lookup;
1897 			}
1898 			goto discard_and_relse;
1899 		}
1900 		if (nsk == sk) {
1901 			reqsk_put(req);
1902 			tcp_v4_restore_cb(skb);
1903 		} else if (tcp_child_process(sk, nsk, skb)) {
1904 			tcp_v4_send_reset(nsk, skb);
1905 			goto discard_and_relse;
1906 		} else {
1907 			sock_put(sk);
1908 			return 0;
1909 		}
1910 	}
1911 	if (unlikely(iph->ttl < inet_sk(sk)->min_ttl)) {
1912 		__NET_INC_STATS(net, LINUX_MIB_TCPMINTTLDROP);
1913 		goto discard_and_relse;
1914 	}
1915 
1916 	if (!xfrm4_policy_check(sk, XFRM_POLICY_IN, skb))
1917 		goto discard_and_relse;
1918 
1919 	if (tcp_v4_inbound_md5_hash(sk, skb))
1920 		goto discard_and_relse;
1921 
1922 	nf_reset_ct(skb);
1923 
1924 	if (tcp_filter(sk, skb))
1925 		goto discard_and_relse;
1926 	th = (const struct tcphdr *)skb->data;
1927 	iph = ip_hdr(skb);
1928 	tcp_v4_fill_cb(skb, iph, th);
1929 
1930 	skb->dev = NULL;
1931 
1932 	if (sk->sk_state == TCP_LISTEN) {
1933 		ret = tcp_v4_do_rcv(sk, skb);
1934 		goto put_and_return;
1935 	}
1936 
1937 	sk_incoming_cpu_update(sk);
1938 
1939 	bh_lock_sock_nested(sk);
1940 	tcp_segs_in(tcp_sk(sk), skb);
1941 	ret = 0;
1942 	if (!sock_owned_by_user(sk)) {
1943 		skb_to_free = sk->sk_rx_skb_cache;
1944 		sk->sk_rx_skb_cache = NULL;
1945 		ret = tcp_v4_do_rcv(sk, skb);
1946 	} else {
1947 		if (tcp_add_backlog(sk, skb))
1948 			goto discard_and_relse;
1949 		skb_to_free = NULL;
1950 	}
1951 	bh_unlock_sock(sk);
1952 	if (skb_to_free)
1953 		__kfree_skb(skb_to_free);
1954 
1955 put_and_return:
1956 	if (refcounted)
1957 		sock_put(sk);
1958 
1959 	return ret;
1960 
1961 no_tcp_socket:
1962 	if (!xfrm4_policy_check(NULL, XFRM_POLICY_IN, skb))
1963 		goto discard_it;
1964 
1965 	tcp_v4_fill_cb(skb, iph, th);
1966 
1967 	if (tcp_checksum_complete(skb)) {
1968 csum_error:
1969 		__TCP_INC_STATS(net, TCP_MIB_CSUMERRORS);
1970 bad_packet:
1971 		__TCP_INC_STATS(net, TCP_MIB_INERRS);
1972 	} else {
1973 		tcp_v4_send_reset(NULL, skb);
1974 	}
1975 
1976 discard_it:
1977 	/* Discard frame. */
1978 	kfree_skb(skb);
1979 	return 0;
1980 
1981 discard_and_relse:
1982 	sk_drops_add(sk, skb);
1983 	if (refcounted)
1984 		sock_put(sk);
1985 	goto discard_it;
1986 
1987 do_time_wait:
1988 	if (!xfrm4_policy_check(NULL, XFRM_POLICY_IN, skb)) {
1989 		inet_twsk_put(inet_twsk(sk));
1990 		goto discard_it;
1991 	}
1992 
1993 	tcp_v4_fill_cb(skb, iph, th);
1994 
1995 	if (tcp_checksum_complete(skb)) {
1996 		inet_twsk_put(inet_twsk(sk));
1997 		goto csum_error;
1998 	}
1999 	switch (tcp_timewait_state_process(inet_twsk(sk), skb, th)) {
2000 	case TCP_TW_SYN: {
2001 		struct sock *sk2 = inet_lookup_listener(dev_net(skb->dev),
2002 							&tcp_hashinfo, skb,
2003 							__tcp_hdrlen(th),
2004 							iph->saddr, th->source,
2005 							iph->daddr, th->dest,
2006 							inet_iif(skb),
2007 							sdif);
2008 		if (sk2) {
2009 			inet_twsk_deschedule_put(inet_twsk(sk));
2010 			sk = sk2;
2011 			tcp_v4_restore_cb(skb);
2012 			refcounted = false;
2013 			goto process;
2014 		}
2015 	}
2016 		/* to ACK */
2017 		/* fall through */
2018 	case TCP_TW_ACK:
2019 		tcp_v4_timewait_ack(sk, skb);
2020 		break;
2021 	case TCP_TW_RST:
2022 		tcp_v4_send_reset(sk, skb);
2023 		inet_twsk_deschedule_put(inet_twsk(sk));
2024 		goto discard_it;
2025 	case TCP_TW_SUCCESS:;
2026 	}
2027 	goto discard_it;
2028 }
2029 
2030 static struct timewait_sock_ops tcp_timewait_sock_ops = {
2031 	.twsk_obj_size	= sizeof(struct tcp_timewait_sock),
2032 	.twsk_unique	= tcp_twsk_unique,
2033 	.twsk_destructor= tcp_twsk_destructor,
2034 };
2035 
2036 void inet_sk_rx_dst_set(struct sock *sk, const struct sk_buff *skb)
2037 {
2038 	struct dst_entry *dst = skb_dst(skb);
2039 
2040 	if (dst && dst_hold_safe(dst)) {
2041 		sk->sk_rx_dst = dst;
2042 		inet_sk(sk)->rx_dst_ifindex = skb->skb_iif;
2043 	}
2044 }
2045 EXPORT_SYMBOL(inet_sk_rx_dst_set);
2046 
2047 const struct inet_connection_sock_af_ops ipv4_specific = {
2048 	.queue_xmit	   = ip_queue_xmit,
2049 	.send_check	   = tcp_v4_send_check,
2050 	.rebuild_header	   = inet_sk_rebuild_header,
2051 	.sk_rx_dst_set	   = inet_sk_rx_dst_set,
2052 	.conn_request	   = tcp_v4_conn_request,
2053 	.syn_recv_sock	   = tcp_v4_syn_recv_sock,
2054 	.net_header_len	   = sizeof(struct iphdr),
2055 	.setsockopt	   = ip_setsockopt,
2056 	.getsockopt	   = ip_getsockopt,
2057 	.addr2sockaddr	   = inet_csk_addr2sockaddr,
2058 	.sockaddr_len	   = sizeof(struct sockaddr_in),
2059 #ifdef CONFIG_COMPAT
2060 	.compat_setsockopt = compat_ip_setsockopt,
2061 	.compat_getsockopt = compat_ip_getsockopt,
2062 #endif
2063 	.mtu_reduced	   = tcp_v4_mtu_reduced,
2064 };
2065 EXPORT_SYMBOL(ipv4_specific);
2066 
2067 #ifdef CONFIG_TCP_MD5SIG
2068 static const struct tcp_sock_af_ops tcp_sock_ipv4_specific = {
2069 	.md5_lookup		= tcp_v4_md5_lookup,
2070 	.calc_md5_hash		= tcp_v4_md5_hash_skb,
2071 	.md5_parse		= tcp_v4_parse_md5_keys,
2072 };
2073 #endif
2074 
2075 /* NOTE: A lot of things set to zero explicitly by call to
2076  *       sk_alloc() so need not be done here.
2077  */
2078 static int tcp_v4_init_sock(struct sock *sk)
2079 {
2080 	struct inet_connection_sock *icsk = inet_csk(sk);
2081 
2082 	tcp_init_sock(sk);
2083 
2084 	icsk->icsk_af_ops = &ipv4_specific;
2085 
2086 #ifdef CONFIG_TCP_MD5SIG
2087 	tcp_sk(sk)->af_specific = &tcp_sock_ipv4_specific;
2088 #endif
2089 
2090 	return 0;
2091 }
2092 
2093 void tcp_v4_destroy_sock(struct sock *sk)
2094 {
2095 	struct tcp_sock *tp = tcp_sk(sk);
2096 
2097 	trace_tcp_destroy_sock(sk);
2098 
2099 	tcp_clear_xmit_timers(sk);
2100 
2101 	tcp_cleanup_congestion_control(sk);
2102 
2103 	tcp_cleanup_ulp(sk);
2104 
2105 	/* Cleanup up the write buffer. */
2106 	tcp_write_queue_purge(sk);
2107 
2108 	/* Check if we want to disable active TFO */
2109 	tcp_fastopen_active_disable_ofo_check(sk);
2110 
2111 	/* Cleans up our, hopefully empty, out_of_order_queue. */
2112 	skb_rbtree_purge(&tp->out_of_order_queue);
2113 
2114 #ifdef CONFIG_TCP_MD5SIG
2115 	/* Clean up the MD5 key list, if any */
2116 	if (tp->md5sig_info) {
2117 		tcp_clear_md5_list(sk);
2118 		kfree_rcu(rcu_dereference_protected(tp->md5sig_info, 1), rcu);
2119 		tp->md5sig_info = NULL;
2120 	}
2121 #endif
2122 
2123 	/* Clean up a referenced TCP bind bucket. */
2124 	if (inet_csk(sk)->icsk_bind_hash)
2125 		inet_put_port(sk);
2126 
2127 	BUG_ON(rcu_access_pointer(tp->fastopen_rsk));
2128 
2129 	/* If socket is aborted during connect operation */
2130 	tcp_free_fastopen_req(tp);
2131 	tcp_fastopen_destroy_cipher(sk);
2132 	tcp_saved_syn_free(tp);
2133 
2134 	sk_sockets_allocated_dec(sk);
2135 }
2136 EXPORT_SYMBOL(tcp_v4_destroy_sock);
2137 
2138 #ifdef CONFIG_PROC_FS
2139 /* Proc filesystem TCP sock list dumping. */
2140 
2141 /*
2142  * Get next listener socket follow cur.  If cur is NULL, get first socket
2143  * starting from bucket given in st->bucket; when st->bucket is zero the
2144  * very first socket in the hash table is returned.
2145  */
2146 static void *listening_get_next(struct seq_file *seq, void *cur)
2147 {
2148 	struct tcp_seq_afinfo *afinfo = PDE_DATA(file_inode(seq->file));
2149 	struct tcp_iter_state *st = seq->private;
2150 	struct net *net = seq_file_net(seq);
2151 	struct inet_listen_hashbucket *ilb;
2152 	struct sock *sk = cur;
2153 
2154 	if (!sk) {
2155 get_head:
2156 		ilb = &tcp_hashinfo.listening_hash[st->bucket];
2157 		spin_lock(&ilb->lock);
2158 		sk = sk_head(&ilb->head);
2159 		st->offset = 0;
2160 		goto get_sk;
2161 	}
2162 	ilb = &tcp_hashinfo.listening_hash[st->bucket];
2163 	++st->num;
2164 	++st->offset;
2165 
2166 	sk = sk_next(sk);
2167 get_sk:
2168 	sk_for_each_from(sk) {
2169 		if (!net_eq(sock_net(sk), net))
2170 			continue;
2171 		if (sk->sk_family == afinfo->family)
2172 			return sk;
2173 	}
2174 	spin_unlock(&ilb->lock);
2175 	st->offset = 0;
2176 	if (++st->bucket < INET_LHTABLE_SIZE)
2177 		goto get_head;
2178 	return NULL;
2179 }
2180 
2181 static void *listening_get_idx(struct seq_file *seq, loff_t *pos)
2182 {
2183 	struct tcp_iter_state *st = seq->private;
2184 	void *rc;
2185 
2186 	st->bucket = 0;
2187 	st->offset = 0;
2188 	rc = listening_get_next(seq, NULL);
2189 
2190 	while (rc && *pos) {
2191 		rc = listening_get_next(seq, rc);
2192 		--*pos;
2193 	}
2194 	return rc;
2195 }
2196 
2197 static inline bool empty_bucket(const struct tcp_iter_state *st)
2198 {
2199 	return hlist_nulls_empty(&tcp_hashinfo.ehash[st->bucket].chain);
2200 }
2201 
2202 /*
2203  * Get first established socket starting from bucket given in st->bucket.
2204  * If st->bucket is zero, the very first socket in the hash is returned.
2205  */
2206 static void *established_get_first(struct seq_file *seq)
2207 {
2208 	struct tcp_seq_afinfo *afinfo = PDE_DATA(file_inode(seq->file));
2209 	struct tcp_iter_state *st = seq->private;
2210 	struct net *net = seq_file_net(seq);
2211 	void *rc = NULL;
2212 
2213 	st->offset = 0;
2214 	for (; st->bucket <= tcp_hashinfo.ehash_mask; ++st->bucket) {
2215 		struct sock *sk;
2216 		struct hlist_nulls_node *node;
2217 		spinlock_t *lock = inet_ehash_lockp(&tcp_hashinfo, st->bucket);
2218 
2219 		/* Lockless fast path for the common case of empty buckets */
2220 		if (empty_bucket(st))
2221 			continue;
2222 
2223 		spin_lock_bh(lock);
2224 		sk_nulls_for_each(sk, node, &tcp_hashinfo.ehash[st->bucket].chain) {
2225 			if (sk->sk_family != afinfo->family ||
2226 			    !net_eq(sock_net(sk), net)) {
2227 				continue;
2228 			}
2229 			rc = sk;
2230 			goto out;
2231 		}
2232 		spin_unlock_bh(lock);
2233 	}
2234 out:
2235 	return rc;
2236 }
2237 
2238 static void *established_get_next(struct seq_file *seq, void *cur)
2239 {
2240 	struct tcp_seq_afinfo *afinfo = PDE_DATA(file_inode(seq->file));
2241 	struct sock *sk = cur;
2242 	struct hlist_nulls_node *node;
2243 	struct tcp_iter_state *st = seq->private;
2244 	struct net *net = seq_file_net(seq);
2245 
2246 	++st->num;
2247 	++st->offset;
2248 
2249 	sk = sk_nulls_next(sk);
2250 
2251 	sk_nulls_for_each_from(sk, node) {
2252 		if (sk->sk_family == afinfo->family &&
2253 		    net_eq(sock_net(sk), net))
2254 			return sk;
2255 	}
2256 
2257 	spin_unlock_bh(inet_ehash_lockp(&tcp_hashinfo, st->bucket));
2258 	++st->bucket;
2259 	return established_get_first(seq);
2260 }
2261 
2262 static void *established_get_idx(struct seq_file *seq, loff_t pos)
2263 {
2264 	struct tcp_iter_state *st = seq->private;
2265 	void *rc;
2266 
2267 	st->bucket = 0;
2268 	rc = established_get_first(seq);
2269 
2270 	while (rc && pos) {
2271 		rc = established_get_next(seq, rc);
2272 		--pos;
2273 	}
2274 	return rc;
2275 }
2276 
2277 static void *tcp_get_idx(struct seq_file *seq, loff_t pos)
2278 {
2279 	void *rc;
2280 	struct tcp_iter_state *st = seq->private;
2281 
2282 	st->state = TCP_SEQ_STATE_LISTENING;
2283 	rc	  = listening_get_idx(seq, &pos);
2284 
2285 	if (!rc) {
2286 		st->state = TCP_SEQ_STATE_ESTABLISHED;
2287 		rc	  = established_get_idx(seq, pos);
2288 	}
2289 
2290 	return rc;
2291 }
2292 
2293 static void *tcp_seek_last_pos(struct seq_file *seq)
2294 {
2295 	struct tcp_iter_state *st = seq->private;
2296 	int offset = st->offset;
2297 	int orig_num = st->num;
2298 	void *rc = NULL;
2299 
2300 	switch (st->state) {
2301 	case TCP_SEQ_STATE_LISTENING:
2302 		if (st->bucket >= INET_LHTABLE_SIZE)
2303 			break;
2304 		st->state = TCP_SEQ_STATE_LISTENING;
2305 		rc = listening_get_next(seq, NULL);
2306 		while (offset-- && rc)
2307 			rc = listening_get_next(seq, rc);
2308 		if (rc)
2309 			break;
2310 		st->bucket = 0;
2311 		st->state = TCP_SEQ_STATE_ESTABLISHED;
2312 		/* Fallthrough */
2313 	case TCP_SEQ_STATE_ESTABLISHED:
2314 		if (st->bucket > tcp_hashinfo.ehash_mask)
2315 			break;
2316 		rc = established_get_first(seq);
2317 		while (offset-- && rc)
2318 			rc = established_get_next(seq, rc);
2319 	}
2320 
2321 	st->num = orig_num;
2322 
2323 	return rc;
2324 }
2325 
2326 void *tcp_seq_start(struct seq_file *seq, loff_t *pos)
2327 {
2328 	struct tcp_iter_state *st = seq->private;
2329 	void *rc;
2330 
2331 	if (*pos && *pos == st->last_pos) {
2332 		rc = tcp_seek_last_pos(seq);
2333 		if (rc)
2334 			goto out;
2335 	}
2336 
2337 	st->state = TCP_SEQ_STATE_LISTENING;
2338 	st->num = 0;
2339 	st->bucket = 0;
2340 	st->offset = 0;
2341 	rc = *pos ? tcp_get_idx(seq, *pos - 1) : SEQ_START_TOKEN;
2342 
2343 out:
2344 	st->last_pos = *pos;
2345 	return rc;
2346 }
2347 EXPORT_SYMBOL(tcp_seq_start);
2348 
2349 void *tcp_seq_next(struct seq_file *seq, void *v, loff_t *pos)
2350 {
2351 	struct tcp_iter_state *st = seq->private;
2352 	void *rc = NULL;
2353 
2354 	if (v == SEQ_START_TOKEN) {
2355 		rc = tcp_get_idx(seq, 0);
2356 		goto out;
2357 	}
2358 
2359 	switch (st->state) {
2360 	case TCP_SEQ_STATE_LISTENING:
2361 		rc = listening_get_next(seq, v);
2362 		if (!rc) {
2363 			st->state = TCP_SEQ_STATE_ESTABLISHED;
2364 			st->bucket = 0;
2365 			st->offset = 0;
2366 			rc	  = established_get_first(seq);
2367 		}
2368 		break;
2369 	case TCP_SEQ_STATE_ESTABLISHED:
2370 		rc = established_get_next(seq, v);
2371 		break;
2372 	}
2373 out:
2374 	++*pos;
2375 	st->last_pos = *pos;
2376 	return rc;
2377 }
2378 EXPORT_SYMBOL(tcp_seq_next);
2379 
2380 void tcp_seq_stop(struct seq_file *seq, void *v)
2381 {
2382 	struct tcp_iter_state *st = seq->private;
2383 
2384 	switch (st->state) {
2385 	case TCP_SEQ_STATE_LISTENING:
2386 		if (v != SEQ_START_TOKEN)
2387 			spin_unlock(&tcp_hashinfo.listening_hash[st->bucket].lock);
2388 		break;
2389 	case TCP_SEQ_STATE_ESTABLISHED:
2390 		if (v)
2391 			spin_unlock_bh(inet_ehash_lockp(&tcp_hashinfo, st->bucket));
2392 		break;
2393 	}
2394 }
2395 EXPORT_SYMBOL(tcp_seq_stop);
2396 
2397 static void get_openreq4(const struct request_sock *req,
2398 			 struct seq_file *f, int i)
2399 {
2400 	const struct inet_request_sock *ireq = inet_rsk(req);
2401 	long delta = req->rsk_timer.expires - jiffies;
2402 
2403 	seq_printf(f, "%4d: %08X:%04X %08X:%04X"
2404 		" %02X %08X:%08X %02X:%08lX %08X %5u %8d %u %d %pK",
2405 		i,
2406 		ireq->ir_loc_addr,
2407 		ireq->ir_num,
2408 		ireq->ir_rmt_addr,
2409 		ntohs(ireq->ir_rmt_port),
2410 		TCP_SYN_RECV,
2411 		0, 0, /* could print option size, but that is af dependent. */
2412 		1,    /* timers active (only the expire timer) */
2413 		jiffies_delta_to_clock_t(delta),
2414 		req->num_timeout,
2415 		from_kuid_munged(seq_user_ns(f),
2416 				 sock_i_uid(req->rsk_listener)),
2417 		0,  /* non standard timer */
2418 		0, /* open_requests have no inode */
2419 		0,
2420 		req);
2421 }
2422 
2423 static void get_tcp4_sock(struct sock *sk, struct seq_file *f, int i)
2424 {
2425 	int timer_active;
2426 	unsigned long timer_expires;
2427 	const struct tcp_sock *tp = tcp_sk(sk);
2428 	const struct inet_connection_sock *icsk = inet_csk(sk);
2429 	const struct inet_sock *inet = inet_sk(sk);
2430 	const struct fastopen_queue *fastopenq = &icsk->icsk_accept_queue.fastopenq;
2431 	__be32 dest = inet->inet_daddr;
2432 	__be32 src = inet->inet_rcv_saddr;
2433 	__u16 destp = ntohs(inet->inet_dport);
2434 	__u16 srcp = ntohs(inet->inet_sport);
2435 	int rx_queue;
2436 	int state;
2437 
2438 	if (icsk->icsk_pending == ICSK_TIME_RETRANS ||
2439 	    icsk->icsk_pending == ICSK_TIME_REO_TIMEOUT ||
2440 	    icsk->icsk_pending == ICSK_TIME_LOSS_PROBE) {
2441 		timer_active	= 1;
2442 		timer_expires	= icsk->icsk_timeout;
2443 	} else if (icsk->icsk_pending == ICSK_TIME_PROBE0) {
2444 		timer_active	= 4;
2445 		timer_expires	= icsk->icsk_timeout;
2446 	} else if (timer_pending(&sk->sk_timer)) {
2447 		timer_active	= 2;
2448 		timer_expires	= sk->sk_timer.expires;
2449 	} else {
2450 		timer_active	= 0;
2451 		timer_expires = jiffies;
2452 	}
2453 
2454 	state = inet_sk_state_load(sk);
2455 	if (state == TCP_LISTEN)
2456 		rx_queue = sk->sk_ack_backlog;
2457 	else
2458 		/* Because we don't lock the socket,
2459 		 * we might find a transient negative value.
2460 		 */
2461 		rx_queue = max_t(int, READ_ONCE(tp->rcv_nxt) -
2462 				      READ_ONCE(tp->copied_seq), 0);
2463 
2464 	seq_printf(f, "%4d: %08X:%04X %08X:%04X %02X %08X:%08X %02X:%08lX "
2465 			"%08X %5u %8d %lu %d %pK %lu %lu %u %u %d",
2466 		i, src, srcp, dest, destp, state,
2467 		READ_ONCE(tp->write_seq) - tp->snd_una,
2468 		rx_queue,
2469 		timer_active,
2470 		jiffies_delta_to_clock_t(timer_expires - jiffies),
2471 		icsk->icsk_retransmits,
2472 		from_kuid_munged(seq_user_ns(f), sock_i_uid(sk)),
2473 		icsk->icsk_probes_out,
2474 		sock_i_ino(sk),
2475 		refcount_read(&sk->sk_refcnt), sk,
2476 		jiffies_to_clock_t(icsk->icsk_rto),
2477 		jiffies_to_clock_t(icsk->icsk_ack.ato),
2478 		(icsk->icsk_ack.quick << 1) | inet_csk_in_pingpong_mode(sk),
2479 		tp->snd_cwnd,
2480 		state == TCP_LISTEN ?
2481 		    fastopenq->max_qlen :
2482 		    (tcp_in_initial_slowstart(tp) ? -1 : tp->snd_ssthresh));
2483 }
2484 
2485 static void get_timewait4_sock(const struct inet_timewait_sock *tw,
2486 			       struct seq_file *f, int i)
2487 {
2488 	long delta = tw->tw_timer.expires - jiffies;
2489 	__be32 dest, src;
2490 	__u16 destp, srcp;
2491 
2492 	dest  = tw->tw_daddr;
2493 	src   = tw->tw_rcv_saddr;
2494 	destp = ntohs(tw->tw_dport);
2495 	srcp  = ntohs(tw->tw_sport);
2496 
2497 	seq_printf(f, "%4d: %08X:%04X %08X:%04X"
2498 		" %02X %08X:%08X %02X:%08lX %08X %5d %8d %d %d %pK",
2499 		i, src, srcp, dest, destp, tw->tw_substate, 0, 0,
2500 		3, jiffies_delta_to_clock_t(delta), 0, 0, 0, 0,
2501 		refcount_read(&tw->tw_refcnt), tw);
2502 }
2503 
2504 #define TMPSZ 150
2505 
2506 static int tcp4_seq_show(struct seq_file *seq, void *v)
2507 {
2508 	struct tcp_iter_state *st;
2509 	struct sock *sk = v;
2510 
2511 	seq_setwidth(seq, TMPSZ - 1);
2512 	if (v == SEQ_START_TOKEN) {
2513 		seq_puts(seq, "  sl  local_address rem_address   st tx_queue "
2514 			   "rx_queue tr tm->when retrnsmt   uid  timeout "
2515 			   "inode");
2516 		goto out;
2517 	}
2518 	st = seq->private;
2519 
2520 	if (sk->sk_state == TCP_TIME_WAIT)
2521 		get_timewait4_sock(v, seq, st->num);
2522 	else if (sk->sk_state == TCP_NEW_SYN_RECV)
2523 		get_openreq4(v, seq, st->num);
2524 	else
2525 		get_tcp4_sock(v, seq, st->num);
2526 out:
2527 	seq_pad(seq, '\n');
2528 	return 0;
2529 }
2530 
2531 static const struct seq_operations tcp4_seq_ops = {
2532 	.show		= tcp4_seq_show,
2533 	.start		= tcp_seq_start,
2534 	.next		= tcp_seq_next,
2535 	.stop		= tcp_seq_stop,
2536 };
2537 
2538 static struct tcp_seq_afinfo tcp4_seq_afinfo = {
2539 	.family		= AF_INET,
2540 };
2541 
2542 static int __net_init tcp4_proc_init_net(struct net *net)
2543 {
2544 	if (!proc_create_net_data("tcp", 0444, net->proc_net, &tcp4_seq_ops,
2545 			sizeof(struct tcp_iter_state), &tcp4_seq_afinfo))
2546 		return -ENOMEM;
2547 	return 0;
2548 }
2549 
2550 static void __net_exit tcp4_proc_exit_net(struct net *net)
2551 {
2552 	remove_proc_entry("tcp", net->proc_net);
2553 }
2554 
2555 static struct pernet_operations tcp4_net_ops = {
2556 	.init = tcp4_proc_init_net,
2557 	.exit = tcp4_proc_exit_net,
2558 };
2559 
2560 int __init tcp4_proc_init(void)
2561 {
2562 	return register_pernet_subsys(&tcp4_net_ops);
2563 }
2564 
2565 void tcp4_proc_exit(void)
2566 {
2567 	unregister_pernet_subsys(&tcp4_net_ops);
2568 }
2569 #endif /* CONFIG_PROC_FS */
2570 
2571 struct proto tcp_prot = {
2572 	.name			= "TCP",
2573 	.owner			= THIS_MODULE,
2574 	.close			= tcp_close,
2575 	.pre_connect		= tcp_v4_pre_connect,
2576 	.connect		= tcp_v4_connect,
2577 	.disconnect		= tcp_disconnect,
2578 	.accept			= inet_csk_accept,
2579 	.ioctl			= tcp_ioctl,
2580 	.init			= tcp_v4_init_sock,
2581 	.destroy		= tcp_v4_destroy_sock,
2582 	.shutdown		= tcp_shutdown,
2583 	.setsockopt		= tcp_setsockopt,
2584 	.getsockopt		= tcp_getsockopt,
2585 	.keepalive		= tcp_set_keepalive,
2586 	.recvmsg		= tcp_recvmsg,
2587 	.sendmsg		= tcp_sendmsg,
2588 	.sendpage		= tcp_sendpage,
2589 	.backlog_rcv		= tcp_v4_do_rcv,
2590 	.release_cb		= tcp_release_cb,
2591 	.hash			= inet_hash,
2592 	.unhash			= inet_unhash,
2593 	.get_port		= inet_csk_get_port,
2594 	.enter_memory_pressure	= tcp_enter_memory_pressure,
2595 	.leave_memory_pressure	= tcp_leave_memory_pressure,
2596 	.stream_memory_free	= tcp_stream_memory_free,
2597 	.sockets_allocated	= &tcp_sockets_allocated,
2598 	.orphan_count		= &tcp_orphan_count,
2599 	.memory_allocated	= &tcp_memory_allocated,
2600 	.memory_pressure	= &tcp_memory_pressure,
2601 	.sysctl_mem		= sysctl_tcp_mem,
2602 	.sysctl_wmem_offset	= offsetof(struct net, ipv4.sysctl_tcp_wmem),
2603 	.sysctl_rmem_offset	= offsetof(struct net, ipv4.sysctl_tcp_rmem),
2604 	.max_header		= MAX_TCP_HEADER,
2605 	.obj_size		= sizeof(struct tcp_sock),
2606 	.slab_flags		= SLAB_TYPESAFE_BY_RCU,
2607 	.twsk_prot		= &tcp_timewait_sock_ops,
2608 	.rsk_prot		= &tcp_request_sock_ops,
2609 	.h.hashinfo		= &tcp_hashinfo,
2610 	.no_autobind		= true,
2611 #ifdef CONFIG_COMPAT
2612 	.compat_setsockopt	= compat_tcp_setsockopt,
2613 	.compat_getsockopt	= compat_tcp_getsockopt,
2614 #endif
2615 	.diag_destroy		= tcp_abort,
2616 };
2617 EXPORT_SYMBOL(tcp_prot);
2618 
2619 static void __net_exit tcp_sk_exit(struct net *net)
2620 {
2621 	int cpu;
2622 
2623 	if (net->ipv4.tcp_congestion_control)
2624 		module_put(net->ipv4.tcp_congestion_control->owner);
2625 
2626 	for_each_possible_cpu(cpu)
2627 		inet_ctl_sock_destroy(*per_cpu_ptr(net->ipv4.tcp_sk, cpu));
2628 	free_percpu(net->ipv4.tcp_sk);
2629 }
2630 
2631 static int __net_init tcp_sk_init(struct net *net)
2632 {
2633 	int res, cpu, cnt;
2634 
2635 	net->ipv4.tcp_sk = alloc_percpu(struct sock *);
2636 	if (!net->ipv4.tcp_sk)
2637 		return -ENOMEM;
2638 
2639 	for_each_possible_cpu(cpu) {
2640 		struct sock *sk;
2641 
2642 		res = inet_ctl_sock_create(&sk, PF_INET, SOCK_RAW,
2643 					   IPPROTO_TCP, net);
2644 		if (res)
2645 			goto fail;
2646 		sock_set_flag(sk, SOCK_USE_WRITE_QUEUE);
2647 
2648 		/* Please enforce IP_DF and IPID==0 for RST and
2649 		 * ACK sent in SYN-RECV and TIME-WAIT state.
2650 		 */
2651 		inet_sk(sk)->pmtudisc = IP_PMTUDISC_DO;
2652 
2653 		*per_cpu_ptr(net->ipv4.tcp_sk, cpu) = sk;
2654 	}
2655 
2656 	net->ipv4.sysctl_tcp_ecn = 2;
2657 	net->ipv4.sysctl_tcp_ecn_fallback = 1;
2658 
2659 	net->ipv4.sysctl_tcp_base_mss = TCP_BASE_MSS;
2660 	net->ipv4.sysctl_tcp_min_snd_mss = TCP_MIN_SND_MSS;
2661 	net->ipv4.sysctl_tcp_probe_threshold = TCP_PROBE_THRESHOLD;
2662 	net->ipv4.sysctl_tcp_probe_interval = TCP_PROBE_INTERVAL;
2663 	net->ipv4.sysctl_tcp_mtu_probe_floor = TCP_MIN_SND_MSS;
2664 
2665 	net->ipv4.sysctl_tcp_keepalive_time = TCP_KEEPALIVE_TIME;
2666 	net->ipv4.sysctl_tcp_keepalive_probes = TCP_KEEPALIVE_PROBES;
2667 	net->ipv4.sysctl_tcp_keepalive_intvl = TCP_KEEPALIVE_INTVL;
2668 
2669 	net->ipv4.sysctl_tcp_syn_retries = TCP_SYN_RETRIES;
2670 	net->ipv4.sysctl_tcp_synack_retries = TCP_SYNACK_RETRIES;
2671 	net->ipv4.sysctl_tcp_syncookies = 1;
2672 	net->ipv4.sysctl_tcp_reordering = TCP_FASTRETRANS_THRESH;
2673 	net->ipv4.sysctl_tcp_retries1 = TCP_RETR1;
2674 	net->ipv4.sysctl_tcp_retries2 = TCP_RETR2;
2675 	net->ipv4.sysctl_tcp_orphan_retries = 0;
2676 	net->ipv4.sysctl_tcp_fin_timeout = TCP_FIN_TIMEOUT;
2677 	net->ipv4.sysctl_tcp_notsent_lowat = UINT_MAX;
2678 	net->ipv4.sysctl_tcp_tw_reuse = 2;
2679 
2680 	cnt = tcp_hashinfo.ehash_mask + 1;
2681 	net->ipv4.tcp_death_row.sysctl_max_tw_buckets = cnt / 2;
2682 	net->ipv4.tcp_death_row.hashinfo = &tcp_hashinfo;
2683 
2684 	net->ipv4.sysctl_max_syn_backlog = max(128, cnt / 128);
2685 	net->ipv4.sysctl_tcp_sack = 1;
2686 	net->ipv4.sysctl_tcp_window_scaling = 1;
2687 	net->ipv4.sysctl_tcp_timestamps = 1;
2688 	net->ipv4.sysctl_tcp_early_retrans = 3;
2689 	net->ipv4.sysctl_tcp_recovery = TCP_RACK_LOSS_DETECTION;
2690 	net->ipv4.sysctl_tcp_slow_start_after_idle = 1; /* By default, RFC2861 behavior.  */
2691 	net->ipv4.sysctl_tcp_retrans_collapse = 1;
2692 	net->ipv4.sysctl_tcp_max_reordering = 300;
2693 	net->ipv4.sysctl_tcp_dsack = 1;
2694 	net->ipv4.sysctl_tcp_app_win = 31;
2695 	net->ipv4.sysctl_tcp_adv_win_scale = 1;
2696 	net->ipv4.sysctl_tcp_frto = 2;
2697 	net->ipv4.sysctl_tcp_moderate_rcvbuf = 1;
2698 	/* This limits the percentage of the congestion window which we
2699 	 * will allow a single TSO frame to consume.  Building TSO frames
2700 	 * which are too large can cause TCP streams to be bursty.
2701 	 */
2702 	net->ipv4.sysctl_tcp_tso_win_divisor = 3;
2703 	/* Default TSQ limit of 16 TSO segments */
2704 	net->ipv4.sysctl_tcp_limit_output_bytes = 16 * 65536;
2705 	/* rfc5961 challenge ack rate limiting */
2706 	net->ipv4.sysctl_tcp_challenge_ack_limit = 1000;
2707 	net->ipv4.sysctl_tcp_min_tso_segs = 2;
2708 	net->ipv4.sysctl_tcp_min_rtt_wlen = 300;
2709 	net->ipv4.sysctl_tcp_autocorking = 1;
2710 	net->ipv4.sysctl_tcp_invalid_ratelimit = HZ/2;
2711 	net->ipv4.sysctl_tcp_pacing_ss_ratio = 200;
2712 	net->ipv4.sysctl_tcp_pacing_ca_ratio = 120;
2713 	if (net != &init_net) {
2714 		memcpy(net->ipv4.sysctl_tcp_rmem,
2715 		       init_net.ipv4.sysctl_tcp_rmem,
2716 		       sizeof(init_net.ipv4.sysctl_tcp_rmem));
2717 		memcpy(net->ipv4.sysctl_tcp_wmem,
2718 		       init_net.ipv4.sysctl_tcp_wmem,
2719 		       sizeof(init_net.ipv4.sysctl_tcp_wmem));
2720 	}
2721 	net->ipv4.sysctl_tcp_comp_sack_delay_ns = NSEC_PER_MSEC;
2722 	net->ipv4.sysctl_tcp_comp_sack_nr = 44;
2723 	net->ipv4.sysctl_tcp_fastopen = TFO_CLIENT_ENABLE;
2724 	spin_lock_init(&net->ipv4.tcp_fastopen_ctx_lock);
2725 	net->ipv4.sysctl_tcp_fastopen_blackhole_timeout = 60 * 60;
2726 	atomic_set(&net->ipv4.tfo_active_disable_times, 0);
2727 
2728 	/* Reno is always built in */
2729 	if (!net_eq(net, &init_net) &&
2730 	    try_module_get(init_net.ipv4.tcp_congestion_control->owner))
2731 		net->ipv4.tcp_congestion_control = init_net.ipv4.tcp_congestion_control;
2732 	else
2733 		net->ipv4.tcp_congestion_control = &tcp_reno;
2734 
2735 	return 0;
2736 fail:
2737 	tcp_sk_exit(net);
2738 
2739 	return res;
2740 }
2741 
2742 static void __net_exit tcp_sk_exit_batch(struct list_head *net_exit_list)
2743 {
2744 	struct net *net;
2745 
2746 	inet_twsk_purge(&tcp_hashinfo, AF_INET);
2747 
2748 	list_for_each_entry(net, net_exit_list, exit_list)
2749 		tcp_fastopen_ctx_destroy(net);
2750 }
2751 
2752 static struct pernet_operations __net_initdata tcp_sk_ops = {
2753        .init	   = tcp_sk_init,
2754        .exit	   = tcp_sk_exit,
2755        .exit_batch = tcp_sk_exit_batch,
2756 };
2757 
2758 void __init tcp_v4_init(void)
2759 {
2760 	if (register_pernet_subsys(&tcp_sk_ops))
2761 		panic("Failed to create the TCP control socket.\n");
2762 }
2763