12874c5fdSThomas Gleixner // SPDX-License-Identifier: GPL-2.0-or-later
21da177e4SLinus Torvalds /*
31da177e4SLinus Torvalds * INET An implementation of the TCP/IP protocol suite for the LINUX
41da177e4SLinus Torvalds * operating system. INET is implemented using the BSD Socket
51da177e4SLinus Torvalds * interface as the means of communication with the user level.
61da177e4SLinus Torvalds *
71da177e4SLinus Torvalds * PF_INET protocol family socket handler.
81da177e4SLinus Torvalds *
902c30a84SJesper Juhl * Authors: Ross Biro
101da177e4SLinus Torvalds * Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
111da177e4SLinus Torvalds * Florian La Roche, <flla@stud.uni-sb.de>
121da177e4SLinus Torvalds * Alan Cox, <A.Cox@swansea.ac.uk>
131da177e4SLinus Torvalds *
141da177e4SLinus Torvalds * Changes (see also sock.c)
151da177e4SLinus Torvalds *
161da177e4SLinus Torvalds * piggy,
171da177e4SLinus Torvalds * Karl Knutson : Socket protocol table
181da177e4SLinus Torvalds * A.N.Kuznetsov : Socket death error in accept().
191da177e4SLinus Torvalds * John Richardson : Fix non blocking error in connect()
201da177e4SLinus Torvalds * so sockets that fail to connect
211da177e4SLinus Torvalds * don't return -EINPROGRESS.
221da177e4SLinus Torvalds * Alan Cox : Asynchronous I/O support
231da177e4SLinus Torvalds * Alan Cox : Keep correct socket pointer on sock
241da177e4SLinus Torvalds * structures
251da177e4SLinus Torvalds * when accept() ed
261da177e4SLinus Torvalds * Alan Cox : Semantics of SO_LINGER aren't state
271da177e4SLinus Torvalds * moved to close when you look carefully.
281da177e4SLinus Torvalds * With this fixed and the accept bug fixed
291da177e4SLinus Torvalds * some RPC stuff seems happier.
301da177e4SLinus Torvalds * Niibe Yutaka : 4.4BSD style write async I/O
311da177e4SLinus Torvalds * Alan Cox,
321da177e4SLinus Torvalds * Tony Gale : Fixed reuse semantics.
331da177e4SLinus Torvalds * Alan Cox : bind() shouldn't abort existing but dead
341da177e4SLinus Torvalds * sockets. Stops FTP netin:.. I hope.
351da177e4SLinus Torvalds * Alan Cox : bind() works correctly for RAW sockets.
361da177e4SLinus Torvalds * Note that FreeBSD at least was broken
371da177e4SLinus Torvalds * in this respect so be careful with
381da177e4SLinus Torvalds * compatibility tests...
391da177e4SLinus Torvalds * Alan Cox : routing cache support
401da177e4SLinus Torvalds * Alan Cox : memzero the socket structure for
411da177e4SLinus Torvalds * compactness.
421da177e4SLinus Torvalds * Matt Day : nonblock connect error handler
431da177e4SLinus Torvalds * Alan Cox : Allow large numbers of pending sockets
441da177e4SLinus Torvalds * (eg for big web sites), but only if
451da177e4SLinus Torvalds * specifically application requested.
461da177e4SLinus Torvalds * Alan Cox : New buffering throughout IP. Used
471da177e4SLinus Torvalds * dumbly.
481da177e4SLinus Torvalds * Alan Cox : New buffering now used smartly.
491da177e4SLinus Torvalds * Alan Cox : BSD rather than common sense
501da177e4SLinus Torvalds * interpretation of listen.
511da177e4SLinus Torvalds * Germano Caronni : Assorted small races.
521da177e4SLinus Torvalds * Alan Cox : sendmsg/recvmsg basic support.
531da177e4SLinus Torvalds * Alan Cox : Only sendmsg/recvmsg now supported.
541da177e4SLinus Torvalds * Alan Cox : Locked down bind (see security list).
551da177e4SLinus Torvalds * Alan Cox : Loosened bind a little.
561da177e4SLinus Torvalds * Mike McLagan : ADD/DEL DLCI Ioctls
571da177e4SLinus Torvalds * Willy Konynenberg : Transparent proxying support.
581da177e4SLinus Torvalds * David S. Miller : New socket lookup architecture.
591da177e4SLinus Torvalds * Some other random speedups.
601da177e4SLinus Torvalds * Cyrus Durgin : Cleaned up file for kmod hacks.
611da177e4SLinus Torvalds * Andi Kleen : Fix inet_stream_connect TCP race.
621da177e4SLinus Torvalds */
631da177e4SLinus Torvalds
64afd46503SJoe Perches #define pr_fmt(fmt) "IPv4: " fmt
65afd46503SJoe Perches
66f4c50d99SHerbert Xu #include <linux/err.h>
671da177e4SLinus Torvalds #include <linux/errno.h>
681da177e4SLinus Torvalds #include <linux/types.h>
691da177e4SLinus Torvalds #include <linux/socket.h>
701da177e4SLinus Torvalds #include <linux/in.h>
711da177e4SLinus Torvalds #include <linux/kernel.h>
72d3fc0353SPaul Gortmaker #include <linux/kmod.h>
731da177e4SLinus Torvalds #include <linux/sched.h>
741da177e4SLinus Torvalds #include <linux/timer.h>
751da177e4SLinus Torvalds #include <linux/string.h>
761da177e4SLinus Torvalds #include <linux/sockios.h>
771da177e4SLinus Torvalds #include <linux/net.h>
784fc268d2SRandy Dunlap #include <linux/capability.h>
791da177e4SLinus Torvalds #include <linux/fcntl.h>
801da177e4SLinus Torvalds #include <linux/mm.h>
811da177e4SLinus Torvalds #include <linux/interrupt.h>
821da177e4SLinus Torvalds #include <linux/stat.h>
831da177e4SLinus Torvalds #include <linux/init.h>
841da177e4SLinus Torvalds #include <linux/poll.h>
851da177e4SLinus Torvalds #include <linux/netfilter_ipv4.h>
86b3da2cf3SDavid S. Miller #include <linux/random.h>
875a0e3ad6STejun Heo #include <linux/slab.h>
881da177e4SLinus Torvalds
897c0f6ba6SLinus Torvalds #include <linux/uaccess.h>
901da177e4SLinus Torvalds
911da177e4SLinus Torvalds #include <linux/inet.h>
921da177e4SLinus Torvalds #include <linux/igmp.h>
9314c85021SArnaldo Carvalho de Melo #include <linux/inetdevice.h>
941da177e4SLinus Torvalds #include <linux/netdevice.h>
9573cc19f1SHerbert Xu #include <net/checksum.h>
961da177e4SLinus Torvalds #include <net/ip.h>
971da177e4SLinus Torvalds #include <net/protocol.h>
981da177e4SLinus Torvalds #include <net/arp.h>
991da177e4SLinus Torvalds #include <net/route.h>
1001da177e4SLinus Torvalds #include <net/ip_fib.h>
101295f7324SArnaldo Carvalho de Melo #include <net/inet_connection_sock.h>
1024721031cSEric Dumazet #include <net/gro.h>
103d457a0e3SEric Dumazet #include <net/gso.h>
1041da177e4SLinus Torvalds #include <net/tcp.h>
1051da177e4SLinus Torvalds #include <net/udp.h>
106ba4e58ecSGerrit Renker #include <net/udplite.h>
107c319b4d7SVasiliy Kulikov #include <net/ping.h>
1081da177e4SLinus Torvalds #include <linux/skbuff.h>
1091da177e4SLinus Torvalds #include <net/sock.h>
1101da177e4SLinus Torvalds #include <net/raw.h>
1111da177e4SLinus Torvalds #include <net/icmp.h>
1121da177e4SLinus Torvalds #include <net/inet_common.h>
113045a0fa0SThomas Graf #include <net/ip_tunnels.h>
1141da177e4SLinus Torvalds #include <net/xfrm.h>
1159b4661bdSPavel Emelyanov #include <net/net_namespace.h>
116aebda156SEric Dumazet #include <net/secure_seq.h>
1171da177e4SLinus Torvalds #ifdef CONFIG_IP_MROUTE
1181da177e4SLinus Torvalds #include <linux/mroute.h>
1191da177e4SLinus Torvalds #endif
1203236b004SDavid Ahern #include <net/l3mdev.h>
121dc13c876SChristoph Hellwig #include <net/compat.h>
1221da177e4SLinus Torvalds
123563e0bb0SYafang Shao #include <trace/events/sock.h>
1241da177e4SLinus Torvalds
1251da177e4SLinus Torvalds /* The inetsw table contains everything that inet_create needs to
1261da177e4SLinus Torvalds * build a new socket.
1271da177e4SLinus Torvalds */
1281da177e4SLinus Torvalds static struct list_head inetsw[SOCK_MAX];
1291da177e4SLinus Torvalds static DEFINE_SPINLOCK(inetsw_lock);
1301da177e4SLinus Torvalds
1311da177e4SLinus Torvalds /* New destruction routine */
1321da177e4SLinus Torvalds
inet_sock_destruct(struct sock * sk)1331da177e4SLinus Torvalds void inet_sock_destruct(struct sock *sk)
1341da177e4SLinus Torvalds {
1351da177e4SLinus Torvalds struct inet_sock *inet = inet_sk(sk);
1361da177e4SLinus Torvalds
1371da177e4SLinus Torvalds __skb_queue_purge(&sk->sk_receive_queue);
1381da177e4SLinus Torvalds __skb_queue_purge(&sk->sk_error_queue);
1391da177e4SLinus Torvalds
1402bb2f5fbSWei Wang sk_mem_reclaim_final(sk);
14195766fffSHideo Aoki
1421da177e4SLinus Torvalds if (sk->sk_type == SOCK_STREAM && sk->sk_state != TCP_CLOSE) {
1433d1427f8SEric Dumazet pr_err("Attempt to release TCP socket in state %d %p\n",
1441da177e4SLinus Torvalds sk->sk_state, sk);
1451da177e4SLinus Torvalds return;
1461da177e4SLinus Torvalds }
1471da177e4SLinus Torvalds if (!sock_flag(sk, SOCK_DEAD)) {
1483d1427f8SEric Dumazet pr_err("Attempt to release alive inet socket %p\n", sk);
1491da177e4SLinus Torvalds return;
1501da177e4SLinus Torvalds }
1511da177e4SLinus Torvalds
1523e7f2b8dSEric Dumazet WARN_ON_ONCE(atomic_read(&sk->sk_rmem_alloc));
1533e7f2b8dSEric Dumazet WARN_ON_ONCE(refcount_read(&sk->sk_wmem_alloc));
1543e7f2b8dSEric Dumazet WARN_ON_ONCE(sk->sk_wmem_queued);
1553e7f2b8dSEric Dumazet WARN_ON_ONCE(sk_forward_alloc_get(sk));
1561da177e4SLinus Torvalds
157f6d8bd05SEric Dumazet kfree(rcu_dereference_protected(inet->inet_opt, 1));
1585869b8faSXin Long dst_release(rcu_dereference_protected(sk->sk_dst_cache, 1));
1598f905c0eSEric Dumazet dst_release(rcu_dereference_protected(sk->sk_rx_dst, 1));
1601da177e4SLinus Torvalds }
1613d1427f8SEric Dumazet EXPORT_SYMBOL(inet_sock_destruct);
1621da177e4SLinus Torvalds
1631da177e4SLinus Torvalds /*
1641da177e4SLinus Torvalds * The routines beyond this point handle the behaviour of an AF_INET
1651da177e4SLinus Torvalds * socket object. Mostly it punts to the subprotocols of IP to do
1661da177e4SLinus Torvalds * the work.
1671da177e4SLinus Torvalds */
1681da177e4SLinus Torvalds
1691da177e4SLinus Torvalds /*
1701da177e4SLinus Torvalds * Automatically bind an unbound socket.
1711da177e4SLinus Torvalds */
1721da177e4SLinus Torvalds
inet_autobind(struct sock * sk)1731da177e4SLinus Torvalds static int inet_autobind(struct sock *sk)
1741da177e4SLinus Torvalds {
1751da177e4SLinus Torvalds struct inet_sock *inet;
1761da177e4SLinus Torvalds /* We may need to bind the socket. */
1771da177e4SLinus Torvalds lock_sock(sk);
1781da177e4SLinus Torvalds inet = inet_sk(sk);
179c720c7e8SEric Dumazet if (!inet->inet_num) {
1801da177e4SLinus Torvalds if (sk->sk_prot->get_port(sk, 0)) {
1811da177e4SLinus Torvalds release_sock(sk);
1821da177e4SLinus Torvalds return -EAGAIN;
1831da177e4SLinus Torvalds }
184c720c7e8SEric Dumazet inet->inet_sport = htons(inet->inet_num);
1851da177e4SLinus Torvalds }
1861da177e4SLinus Torvalds release_sock(sk);
1871da177e4SLinus Torvalds return 0;
1881da177e4SLinus Torvalds }
1891da177e4SLinus Torvalds
__inet_listen_sk(struct sock * sk,int backlog)19071a9a874SPaolo Abeni int __inet_listen_sk(struct sock *sk, int backlog)
1911da177e4SLinus Torvalds {
19271a9a874SPaolo Abeni unsigned char old_state = sk->sk_state;
193e1cfcbe8SHaishuang Yan int err, tcp_fastopen;
1941da177e4SLinus Torvalds
1951da177e4SLinus Torvalds if (!((1 << old_state) & (TCPF_CLOSE | TCPF_LISTEN)))
19671a9a874SPaolo Abeni return -EINVAL;
1971da177e4SLinus Torvalds
198099ecf59SEric Dumazet WRITE_ONCE(sk->sk_max_ack_backlog, backlog);
1991da177e4SLinus Torvalds /* Really, if the socket is already in listen state
2001da177e4SLinus Torvalds * we can only allow the backlog to be adjusted.
2011da177e4SLinus Torvalds */
2021da177e4SLinus Torvalds if (old_state != TCP_LISTEN) {
203cebc5cbaSYuchung Cheng /* Enable TFO w/o requiring TCP_FASTOPEN socket option.
2048336886fSJerry Chu * Note that only TCP sockets (SOCK_STREAM) will reach here.
205cebc5cbaSYuchung Cheng * Also fastopen backlog may already been set via the option
206cebc5cbaSYuchung Cheng * because the socket was in TCP_LISTEN state previously but
207cebc5cbaSYuchung Cheng * was shutdown() rather than close().
2088336886fSJerry Chu */
2095a542133SKuniyuki Iwashima tcp_fastopen = READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_fastopen);
210e1cfcbe8SHaishuang Yan if ((tcp_fastopen & TFO_SERVER_WO_SOCKOPT1) &&
211e1cfcbe8SHaishuang Yan (tcp_fastopen & TFO_SERVER_ENABLE) &&
2120536fcc0SEric Dumazet !inet_csk(sk)->icsk_accept_queue.fastopenq.max_qlen) {
2130536fcc0SEric Dumazet fastopen_queue_tune(sk, backlog);
21443713848SHaishuang Yan tcp_fastopen_init_key_once(sock_net(sk));
2158336886fSJerry Chu }
216cebc5cbaSYuchung Cheng
217e7049395SKuniyuki Iwashima err = inet_csk_listen_start(sk);
2181da177e4SLinus Torvalds if (err)
21971a9a874SPaolo Abeni return err;
22071a9a874SPaolo Abeni
221f333ee0cSAndrey Ignatov tcp_call_bpf(sk, BPF_SOCK_OPS_TCP_LISTEN_CB, 0, NULL);
2221da177e4SLinus Torvalds }
22371a9a874SPaolo Abeni return 0;
22471a9a874SPaolo Abeni }
22571a9a874SPaolo Abeni
22671a9a874SPaolo Abeni /*
22771a9a874SPaolo Abeni * Move a socket into listening state.
22871a9a874SPaolo Abeni */
inet_listen(struct socket * sock,int backlog)22971a9a874SPaolo Abeni int inet_listen(struct socket *sock, int backlog)
23071a9a874SPaolo Abeni {
23171a9a874SPaolo Abeni struct sock *sk = sock->sk;
23271a9a874SPaolo Abeni int err = -EINVAL;
23371a9a874SPaolo Abeni
23471a9a874SPaolo Abeni lock_sock(sk);
23571a9a874SPaolo Abeni
23671a9a874SPaolo Abeni if (sock->state != SS_UNCONNECTED || sock->type != SOCK_STREAM)
23771a9a874SPaolo Abeni goto out;
23871a9a874SPaolo Abeni
23971a9a874SPaolo Abeni err = __inet_listen_sk(sk, backlog);
2401da177e4SLinus Torvalds
2411da177e4SLinus Torvalds out:
2421da177e4SLinus Torvalds release_sock(sk);
2431da177e4SLinus Torvalds return err;
2441da177e4SLinus Torvalds }
2453d1427f8SEric Dumazet EXPORT_SYMBOL(inet_listen);
2461da177e4SLinus Torvalds
2471da177e4SLinus Torvalds /*
2481da177e4SLinus Torvalds * Create an inet socket.
2491da177e4SLinus Torvalds */
2501da177e4SLinus Torvalds
inet_create(struct net * net,struct socket * sock,int protocol,int kern)2513f378b68SEric Paris static int inet_create(struct net *net, struct socket *sock, int protocol,
2523f378b68SEric Paris int kern)
2531da177e4SLinus Torvalds {
2541da177e4SLinus Torvalds struct sock *sk;
2551da177e4SLinus Torvalds struct inet_protosw *answer;
2561da177e4SLinus Torvalds struct inet_sock *inet;
2571da177e4SLinus Torvalds struct proto *answer_prot;
2581da177e4SLinus Torvalds unsigned char answer_flags;
259bb97d31fSArnaldo Carvalho de Melo int try_loading_module = 0;
26086c8f9d1SHerbert Xu int err;
2611da177e4SLinus Torvalds
26279462ad0SHannes Frederic Sowa if (protocol < 0 || protocol >= IPPROTO_MAX)
26379462ad0SHannes Frederic Sowa return -EINVAL;
26479462ad0SHannes Frederic Sowa
2651da177e4SLinus Torvalds sock->state = SS_UNCONNECTED;
2661da177e4SLinus Torvalds
2671da177e4SLinus Torvalds /* Look for the requested type/protocol pair. */
268bb97d31fSArnaldo Carvalho de Melo lookup_protocol:
26986c8f9d1SHerbert Xu err = -ESOCKTNOSUPPORT;
2701da177e4SLinus Torvalds rcu_read_lock();
271696adfe8SPaul E. McKenney list_for_each_entry_rcu(answer, &inetsw[sock->type], list) {
2721da177e4SLinus Torvalds
273696adfe8SPaul E. McKenney err = 0;
2741da177e4SLinus Torvalds /* Check the non-wild match. */
2751da177e4SLinus Torvalds if (protocol == answer->protocol) {
2761da177e4SLinus Torvalds if (protocol != IPPROTO_IP)
2771da177e4SLinus Torvalds break;
2781da177e4SLinus Torvalds } else {
2791da177e4SLinus Torvalds /* Check for the two wild cases. */
2801da177e4SLinus Torvalds if (IPPROTO_IP == protocol) {
2811da177e4SLinus Torvalds protocol = answer->protocol;
2821da177e4SLinus Torvalds break;
2831da177e4SLinus Torvalds }
2841da177e4SLinus Torvalds if (IPPROTO_IP == answer->protocol)
2851da177e4SLinus Torvalds break;
2861da177e4SLinus Torvalds }
28786c8f9d1SHerbert Xu err = -EPROTONOSUPPORT;
2881da177e4SLinus Torvalds }
2891da177e4SLinus Torvalds
290696adfe8SPaul E. McKenney if (unlikely(err)) {
291bb97d31fSArnaldo Carvalho de Melo if (try_loading_module < 2) {
292bb97d31fSArnaldo Carvalho de Melo rcu_read_unlock();
293bb97d31fSArnaldo Carvalho de Melo /*
294bb97d31fSArnaldo Carvalho de Melo * Be more specific, e.g. net-pf-2-proto-132-type-1
295bb97d31fSArnaldo Carvalho de Melo * (net-pf-PF_INET-proto-IPPROTO_SCTP-type-SOCK_STREAM)
296bb97d31fSArnaldo Carvalho de Melo */
297bb97d31fSArnaldo Carvalho de Melo if (++try_loading_module == 1)
298bb97d31fSArnaldo Carvalho de Melo request_module("net-pf-%d-proto-%d-type-%d",
299bb97d31fSArnaldo Carvalho de Melo PF_INET, protocol, sock->type);
300bb97d31fSArnaldo Carvalho de Melo /*
301bb97d31fSArnaldo Carvalho de Melo * Fall back to generic, e.g. net-pf-2-proto-132
302bb97d31fSArnaldo Carvalho de Melo * (net-pf-PF_INET-proto-IPPROTO_SCTP)
303bb97d31fSArnaldo Carvalho de Melo */
304bb97d31fSArnaldo Carvalho de Melo else
305bb97d31fSArnaldo Carvalho de Melo request_module("net-pf-%d-proto-%d",
306bb97d31fSArnaldo Carvalho de Melo PF_INET, protocol);
307bb97d31fSArnaldo Carvalho de Melo goto lookup_protocol;
308bb97d31fSArnaldo Carvalho de Melo } else
3091da177e4SLinus Torvalds goto out_rcu_unlock;
310bb97d31fSArnaldo Carvalho de Melo }
311bb97d31fSArnaldo Carvalho de Melo
3121da177e4SLinus Torvalds err = -EPERM;
31352e804c6SEric W. Biederman if (sock->type == SOCK_RAW && !kern &&
31452e804c6SEric W. Biederman !ns_capable(net->user_ns, CAP_NET_RAW))
3151da177e4SLinus Torvalds goto out_rcu_unlock;
3161da177e4SLinus Torvalds
3171da177e4SLinus Torvalds sock->ops = answer->ops;
3181da177e4SLinus Torvalds answer_prot = answer->prot;
3191da177e4SLinus Torvalds answer_flags = answer->flags;
3201da177e4SLinus Torvalds rcu_read_unlock();
3211da177e4SLinus Torvalds
32251456b29SIan Morris WARN_ON(!answer_prot->slab);
3231da177e4SLinus Torvalds
324ca746c55SZheng Yongjun err = -ENOMEM;
32511aa9c28SEric W. Biederman sk = sk_alloc(net, PF_INET, GFP_KERNEL, answer_prot, kern);
32651456b29SIan Morris if (!sk)
3271da177e4SLinus Torvalds goto out;
3281da177e4SLinus Torvalds
3291da177e4SLinus Torvalds err = 0;
3301da177e4SLinus Torvalds if (INET_PROTOSW_REUSE & answer_flags)
3314a17fd52SPavel Emelyanov sk->sk_reuse = SK_CAN_REUSE;
3321da177e4SLinus Torvalds
333168e7e59SZhengchao Shao if (INET_PROTOSW_ICSK & answer_flags)
334168e7e59SZhengchao Shao inet_init_csk_locks(sk);
335168e7e59SZhengchao Shao
3361da177e4SLinus Torvalds inet = inet_sk(sk);
337b1c0356aSEric Dumazet inet_assign_bit(IS_ICSK, sk, INET_PROTOSW_ICSK & answer_flags);
3381da177e4SLinus Torvalds
339f04b8d34SEric Dumazet inet_clear_bit(NODEFRAG, sk);
3407b2ff18eSJiri Olsa
3411da177e4SLinus Torvalds if (SOCK_RAW == sock->type) {
342c720c7e8SEric Dumazet inet->inet_num = protocol;
3431da177e4SLinus Torvalds if (IPPROTO_RAW == protocol)
344cafbe182SEric Dumazet inet_set_bit(HDRINCL, sk);
3451da177e4SLinus Torvalds }
3461da177e4SLinus Torvalds
3470968d2a4SKuniyuki Iwashima if (READ_ONCE(net->ipv4.sysctl_ip_no_pmtu_disc))
3481da177e4SLinus Torvalds inet->pmtudisc = IP_PMTUDISC_DONT;
3491da177e4SLinus Torvalds else
3501da177e4SLinus Torvalds inet->pmtudisc = IP_PMTUDISC_WANT;
3511da177e4SLinus Torvalds
352f866fbc8SEric Dumazet atomic_set(&inet->inet_id, 0);
3531da177e4SLinus Torvalds
3541da177e4SLinus Torvalds sock_init_data(sock, sk);
3551da177e4SLinus Torvalds
3561da177e4SLinus Torvalds sk->sk_destruct = inet_sock_destruct;
3571da177e4SLinus Torvalds sk->sk_protocol = protocol;
3581da177e4SLinus Torvalds sk->sk_backlog_rcv = sk->sk_prot->backlog_rcv;
359c11204c7SKevin Yang sk->sk_txrehash = READ_ONCE(net->core.sysctl_txrehash);
3601da177e4SLinus Torvalds
3611da177e4SLinus Torvalds inet->uc_ttl = -1;
362b09bde5cSEric Dumazet inet_set_bit(MC_LOOP, sk);
3631da177e4SLinus Torvalds inet->mc_ttl = 1;
364307b4ac6SEric Dumazet inet_set_bit(MC_ALL, sk);
3651da177e4SLinus Torvalds inet->mc_index = 0;
3661da177e4SLinus Torvalds inet->mc_list = NULL;
3674c507d28SJiri Benc inet->rcv_tos = 0;
3681da177e4SLinus Torvalds
369c720c7e8SEric Dumazet if (inet->inet_num) {
3701da177e4SLinus Torvalds /* It assumes that any protocol which allows
3711da177e4SLinus Torvalds * the user to assign a number at socket
3721da177e4SLinus Torvalds * creation time automatically
3731da177e4SLinus Torvalds * shares.
3741da177e4SLinus Torvalds */
375c720c7e8SEric Dumazet inet->inet_sport = htons(inet->inet_num);
3761da177e4SLinus Torvalds /* Add to protocol hash chains. */
377086c653fSCraig Gallek err = sk->sk_prot->hash(sk);
378086c653fSCraig Gallek if (err) {
379086c653fSCraig Gallek sk_common_release(sk);
380086c653fSCraig Gallek goto out;
381086c653fSCraig Gallek }
3821da177e4SLinus Torvalds }
3831da177e4SLinus Torvalds
3841da177e4SLinus Torvalds if (sk->sk_prot->init) {
3851da177e4SLinus Torvalds err = sk->sk_prot->init(sk);
38661023658SDavid Ahern if (err) {
3871da177e4SLinus Torvalds sk_common_release(sk);
38861023658SDavid Ahern goto out;
38961023658SDavid Ahern }
39061023658SDavid Ahern }
39161023658SDavid Ahern
39261023658SDavid Ahern if (!kern) {
39361023658SDavid Ahern err = BPF_CGROUP_RUN_PROG_INET_SOCK(sk);
39461023658SDavid Ahern if (err) {
39561023658SDavid Ahern sk_common_release(sk);
39661023658SDavid Ahern goto out;
39761023658SDavid Ahern }
3981da177e4SLinus Torvalds }
3991da177e4SLinus Torvalds out:
4001da177e4SLinus Torvalds return err;
4011da177e4SLinus Torvalds out_rcu_unlock:
4021da177e4SLinus Torvalds rcu_read_unlock();
4031da177e4SLinus Torvalds goto out;
4041da177e4SLinus Torvalds }
4051da177e4SLinus Torvalds
4061da177e4SLinus Torvalds
4071da177e4SLinus Torvalds /*
4081da177e4SLinus Torvalds * The peer socket should always be NULL (or else). When we call this
4091da177e4SLinus Torvalds * function we are destroying the object and from then on nobody
4101da177e4SLinus Torvalds * should refer to it.
4111da177e4SLinus Torvalds */
inet_release(struct socket * sock)4121da177e4SLinus Torvalds int inet_release(struct socket *sock)
4131da177e4SLinus Torvalds {
4141da177e4SLinus Torvalds struct sock *sk = sock->sk;
4151da177e4SLinus Torvalds
4161da177e4SLinus Torvalds if (sk) {
4171da177e4SLinus Torvalds long timeout;
4181da177e4SLinus Torvalds
419f5836749SStanislav Fomichev if (!sk->sk_kern_sock)
420f5836749SStanislav Fomichev BPF_CGROUP_RUN_PROG_INET_SOCK_RELEASE(sk);
421f5836749SStanislav Fomichev
4221da177e4SLinus Torvalds /* Applications forget to leave groups before exiting */
4231da177e4SLinus Torvalds ip_mc_drop_socket(sk);
4241da177e4SLinus Torvalds
4251da177e4SLinus Torvalds /* If linger is set, we don't return until the close
4261da177e4SLinus Torvalds * is complete. Otherwise we return immediately. The
4271da177e4SLinus Torvalds * actually closing is done the same either way.
4281da177e4SLinus Torvalds *
4291da177e4SLinus Torvalds * If the close is due to the process exiting, we never
4301da177e4SLinus Torvalds * linger..
4311da177e4SLinus Torvalds */
4321da177e4SLinus Torvalds timeout = 0;
4331da177e4SLinus Torvalds if (sock_flag(sk, SOCK_LINGER) &&
4341da177e4SLinus Torvalds !(current->flags & PF_EXITING))
4351da177e4SLinus Torvalds timeout = sk->sk_lingertime;
4361da177e4SLinus Torvalds sk->sk_prot->close(sk, timeout);
4372b81f816SJakub Kicinski sock->sk = NULL;
4381da177e4SLinus Torvalds }
4391da177e4SLinus Torvalds return 0;
4401da177e4SLinus Torvalds }
4413d1427f8SEric Dumazet EXPORT_SYMBOL(inet_release);
4421da177e4SLinus Torvalds
inet_bind_sk(struct sock * sk,struct sockaddr * uaddr,int addr_len)443e6d360ffSPaolo Abeni int inet_bind_sk(struct sock *sk, struct sockaddr *uaddr, int addr_len)
4441da177e4SLinus Torvalds {
44577241217SStanislav Fomichev u32 flags = BIND_WITH_LOCK;
4461da177e4SLinus Torvalds int err;
4471da177e4SLinus Torvalds
4481da177e4SLinus Torvalds /* If the socket has its own bind function then use it. (RAW) */
4491da177e4SLinus Torvalds if (sk->sk_prot->bind) {
4503679d585SAndrey Ignatov return sk->sk_prot->bind(sk, uaddr, addr_len);
4511da177e4SLinus Torvalds }
4521da177e4SLinus Torvalds if (addr_len < sizeof(struct sockaddr_in))
4533679d585SAndrey Ignatov return -EINVAL;
4541da177e4SLinus Torvalds
4554fbac77dSAndrey Ignatov /* BPF prog is run before any checks are done so that if the prog
4564fbac77dSAndrey Ignatov * changes context in a wrong way it will be caught.
4574fbac77dSAndrey Ignatov */
4586d71331eSDaan De Meyer err = BPF_CGROUP_RUN_PROG_INET_BIND_LOCK(sk, uaddr, &addr_len,
4596fc88c35SDave Marchevsky CGROUP_INET4_BIND, &flags);
4604fbac77dSAndrey Ignatov if (err)
4613679d585SAndrey Ignatov return err;
4623679d585SAndrey Ignatov
46377241217SStanislav Fomichev return __inet_bind(sk, uaddr, addr_len, flags);
4643679d585SAndrey Ignatov }
465e6d360ffSPaolo Abeni
inet_bind(struct socket * sock,struct sockaddr * uaddr,int addr_len)466e6d360ffSPaolo Abeni int inet_bind(struct socket *sock, struct sockaddr *uaddr, int addr_len)
467e6d360ffSPaolo Abeni {
468e6d360ffSPaolo Abeni return inet_bind_sk(sock->sk, uaddr, addr_len);
469e6d360ffSPaolo Abeni }
4703679d585SAndrey Ignatov EXPORT_SYMBOL(inet_bind);
4713679d585SAndrey Ignatov
__inet_bind(struct sock * sk,struct sockaddr * uaddr,int addr_len,u32 flags)4723679d585SAndrey Ignatov int __inet_bind(struct sock *sk, struct sockaddr *uaddr, int addr_len,
473cb0721c7SStanislav Fomichev u32 flags)
4743679d585SAndrey Ignatov {
4753679d585SAndrey Ignatov struct sockaddr_in *addr = (struct sockaddr_in *)uaddr;
4761da177e4SLinus Torvalds struct inet_sock *inet = inet_sk(sk);
4771da177e4SLinus Torvalds struct net *net = sock_net(sk);
4781da177e4SLinus Torvalds unsigned short snum;
4791da177e4SLinus Torvalds int chk_addr_ret;
4801da177e4SLinus Torvalds u32 tb_id = RT_TABLE_LOCAL;
4811da177e4SLinus Torvalds int err;
4821da177e4SLinus Torvalds
483c349a528SMarcus Meissner if (addr->sin_family != AF_INET) {
48429c486dfSEric Dumazet /* Compatibility games : accept AF_UNSPEC (mapped to AF_INET)
48529c486dfSEric Dumazet * only if s_addr is INADDR_ANY.
48629c486dfSEric Dumazet */
487c349a528SMarcus Meissner err = -EAFNOSUPPORT;
48829c486dfSEric Dumazet if (addr->sin_family != AF_UNSPEC ||
48929c486dfSEric Dumazet addr->sin_addr.s_addr != htonl(INADDR_ANY))
490d0733d2eSMarcus Meissner goto out;
491c349a528SMarcus Meissner }
492d0733d2eSMarcus Meissner
4933236b004SDavid Ahern tb_id = l3mdev_fib_table_by_index(net, sk->sk_bound_dev_if) ? : tb_id;
49430bbaa19SDavid Ahern chk_addr_ret = inet_addr_type_table(net, addr->sin_addr.s_addr, tb_id);
4951da177e4SLinus Torvalds
4961da177e4SLinus Torvalds /* Not specified by any standard per-se, however it breaks too
4971da177e4SLinus Torvalds * many applications when removed. It is unfortunate since
4981da177e4SLinus Torvalds * allowing applications to make a non-local bind solves
4991da177e4SLinus Torvalds * several problems with systems using dynamic addressing.
5001da177e4SLinus Torvalds * (ie. your servers still start up even if your ISDN link
5011da177e4SLinus Torvalds * is temporarily down)
5021da177e4SLinus Torvalds */
5031da177e4SLinus Torvalds err = -EADDRNOTAVAIL;
5048ff978b8SRiccardo Paolo Bestetti if (!inet_addr_valid_or_nonlocal(net, inet, addr->sin_addr.s_addr,
5058ff978b8SRiccardo Paolo Bestetti chk_addr_ret))
5061da177e4SLinus Torvalds goto out;
5071da177e4SLinus Torvalds
5081da177e4SLinus Torvalds snum = ntohs(addr->sin_port);
5091da177e4SLinus Torvalds err = -EACCES;
51077241217SStanislav Fomichev if (!(flags & BIND_NO_CAP_NET_BIND_SERVICE) &&
51177241217SStanislav Fomichev snum && inet_port_requires_bind_service(net, snum) &&
5123594698aSEric W. Biederman !ns_capable(net->user_ns, CAP_NET_BIND_SERVICE))
5131da177e4SLinus Torvalds goto out;
5141da177e4SLinus Torvalds
5151da177e4SLinus Torvalds /* We keep a pair of addresses. rcv_saddr is the one
5161da177e4SLinus Torvalds * used by hash lookups, and saddr is used for transmit.
5171da177e4SLinus Torvalds *
5181da177e4SLinus Torvalds * In the BSD API these are the same except where it
5191da177e4SLinus Torvalds * would be illegal to use them (multicast/broadcast) in
5201da177e4SLinus Torvalds * which case the sending device address is used.
5211da177e4SLinus Torvalds */
522cb0721c7SStanislav Fomichev if (flags & BIND_WITH_LOCK)
5231da177e4SLinus Torvalds lock_sock(sk);
5241da177e4SLinus Torvalds
5251da177e4SLinus Torvalds /* Check these errors (active socket, double bind). */
5261da177e4SLinus Torvalds err = -EINVAL;
527c720c7e8SEric Dumazet if (sk->sk_state != TCP_CLOSE || inet->inet_num)
5281da177e4SLinus Torvalds goto out_release_sock;
5291da177e4SLinus Torvalds
530c720c7e8SEric Dumazet inet->inet_rcv_saddr = inet->inet_saddr = addr->sin_addr.s_addr;
5311da177e4SLinus Torvalds if (chk_addr_ret == RTN_MULTICAST || chk_addr_ret == RTN_BROADCAST)
532c720c7e8SEric Dumazet inet->inet_saddr = 0; /* Use device */
5331da177e4SLinus Torvalds
5341da177e4SLinus Torvalds /* Make sure we are allowed to bind here. */
535ca571e2eSEric Dumazet if (snum || !(inet_test_bit(BIND_ADDRESS_NO_PORT, sk) ||
536cb0721c7SStanislav Fomichev (flags & BIND_FORCE_ADDRESS_NO_PORT))) {
5377a7160edSKuniyuki Iwashima err = sk->sk_prot->get_port(sk, snum);
5387a7160edSKuniyuki Iwashima if (err) {
539c720c7e8SEric Dumazet inet->inet_saddr = inet->inet_rcv_saddr = 0;
5401da177e4SLinus Torvalds goto out_release_sock;
5411da177e4SLinus Torvalds }
5428086fbafSStanislav Fomichev if (!(flags & BIND_FROM_BPF)) {
543aac3fc32SAndrey Ignatov err = BPF_CGROUP_RUN_PROG_INET4_POST_BIND(sk);
544aac3fc32SAndrey Ignatov if (err) {
545aac3fc32SAndrey Ignatov inet->inet_saddr = inet->inet_rcv_saddr = 0;
54691a760b2SMenglong Dong if (sk->sk_prot->put_port)
54791a760b2SMenglong Dong sk->sk_prot->put_port(sk);
548aac3fc32SAndrey Ignatov goto out_release_sock;
549aac3fc32SAndrey Ignatov }
550aac3fc32SAndrey Ignatov }
5518086fbafSStanislav Fomichev }
5521da177e4SLinus Torvalds
553c720c7e8SEric Dumazet if (inet->inet_rcv_saddr)
5541da177e4SLinus Torvalds sk->sk_userlocks |= SOCK_BINDADDR_LOCK;
5551da177e4SLinus Torvalds if (snum)
5561da177e4SLinus Torvalds sk->sk_userlocks |= SOCK_BINDPORT_LOCK;
557c720c7e8SEric Dumazet inet->inet_sport = htons(inet->inet_num);
558c720c7e8SEric Dumazet inet->inet_daddr = 0;
559c720c7e8SEric Dumazet inet->inet_dport = 0;
5601da177e4SLinus Torvalds sk_dst_reset(sk);
5611da177e4SLinus Torvalds err = 0;
5621da177e4SLinus Torvalds out_release_sock:
563cb0721c7SStanislav Fomichev if (flags & BIND_WITH_LOCK)
5641da177e4SLinus Torvalds release_sock(sk);
5651da177e4SLinus Torvalds out:
5661da177e4SLinus Torvalds return err;
5671da177e4SLinus Torvalds }
5681da177e4SLinus Torvalds
inet_dgram_connect(struct socket * sock,struct sockaddr * uaddr,int addr_len,int flags)5691da177e4SLinus Torvalds int inet_dgram_connect(struct socket *sock, struct sockaddr *uaddr,
5701da177e4SLinus Torvalds int addr_len, int flags)
5711da177e4SLinus Torvalds {
5721da177e4SLinus Torvalds struct sock *sk = sock->sk;
573364f997bSKuniyuki Iwashima const struct proto *prot;
574d74bad4eSAndrey Ignatov int err;
5751da177e4SLinus Torvalds
5766503d961SChangli Gao if (addr_len < sizeof(uaddr->sa_family))
5776503d961SChangli Gao return -EINVAL;
578364f997bSKuniyuki Iwashima
579364f997bSKuniyuki Iwashima /* IPV6_ADDRFORM can change sk->sk_prot under us. */
580364f997bSKuniyuki Iwashima prot = READ_ONCE(sk->sk_prot);
581364f997bSKuniyuki Iwashima
5821da177e4SLinus Torvalds if (uaddr->sa_family == AF_UNSPEC)
583364f997bSKuniyuki Iwashima return prot->disconnect(sk, flags);
5841da177e4SLinus Torvalds
585d74bad4eSAndrey Ignatov if (BPF_CGROUP_PRE_CONNECT_ENABLED(sk)) {
586364f997bSKuniyuki Iwashima err = prot->pre_connect(sk, uaddr, addr_len);
587d74bad4eSAndrey Ignatov if (err)
588d74bad4eSAndrey Ignatov return err;
589d74bad4eSAndrey Ignatov }
590d74bad4eSAndrey Ignatov
591dcd01eeaSEric Dumazet if (data_race(!inet_sk(sk)->inet_num) && inet_autobind(sk))
5921da177e4SLinus Torvalds return -EAGAIN;
593364f997bSKuniyuki Iwashima return prot->connect(sk, uaddr, addr_len);
5941da177e4SLinus Torvalds }
5953d1427f8SEric Dumazet EXPORT_SYMBOL(inet_dgram_connect);
5961da177e4SLinus Torvalds
inet_wait_for_connect(struct sock * sk,long timeo,int writebias)597783237e8SYuchung Cheng static long inet_wait_for_connect(struct sock *sk, long timeo, int writebias)
5981da177e4SLinus Torvalds {
59914135f30SWANG Cong DEFINE_WAIT_FUNC(wait, woken_wake_function);
6001da177e4SLinus Torvalds
60114135f30SWANG Cong add_wait_queue(sk_sleep(sk), &wait);
602783237e8SYuchung Cheng sk->sk_write_pending += writebias;
6031da177e4SLinus Torvalds
6041da177e4SLinus Torvalds /* Basic assumption: if someone sets sk->sk_err, he _must_
6051da177e4SLinus Torvalds * change state of the socket from TCP_SYN_*.
6061da177e4SLinus Torvalds * Connect() does not allow to get error notifications
6071da177e4SLinus Torvalds * without closing the socket.
6081da177e4SLinus Torvalds */
6091da177e4SLinus Torvalds while ((1 << sk->sk_state) & (TCPF_SYN_SENT | TCPF_SYN_RECV)) {
6101da177e4SLinus Torvalds release_sock(sk);
61114135f30SWANG Cong timeo = wait_woken(&wait, TASK_INTERRUPTIBLE, timeo);
6121da177e4SLinus Torvalds lock_sock(sk);
6131da177e4SLinus Torvalds if (signal_pending(current) || !timeo)
6141da177e4SLinus Torvalds break;
6151da177e4SLinus Torvalds }
61614135f30SWANG Cong remove_wait_queue(sk_sleep(sk), &wait);
617783237e8SYuchung Cheng sk->sk_write_pending -= writebias;
6181da177e4SLinus Torvalds return timeo;
6191da177e4SLinus Torvalds }
6201da177e4SLinus Torvalds
6211da177e4SLinus Torvalds /*
6221da177e4SLinus Torvalds * Connect to a remote host. There is regrettably still a little
6231da177e4SLinus Torvalds * TCP 'magic' in here.
6241da177e4SLinus Torvalds */
__inet_stream_connect(struct socket * sock,struct sockaddr * uaddr,int addr_len,int flags,int is_sendmsg)625cf60af03SYuchung Cheng int __inet_stream_connect(struct socket *sock, struct sockaddr *uaddr,
6263979ad7eSWilly Tarreau int addr_len, int flags, int is_sendmsg)
6271da177e4SLinus Torvalds {
6281da177e4SLinus Torvalds struct sock *sk = sock->sk;
6291da177e4SLinus Torvalds int err;
6301da177e4SLinus Torvalds long timeo;
6311da177e4SLinus Torvalds
63219f6d3f3SWei Wang /*
63319f6d3f3SWei Wang * uaddr can be NULL and addr_len can be 0 if:
63419f6d3f3SWei Wang * sk is a TCP fastopen active socket and
63519f6d3f3SWei Wang * TCP_FASTOPEN_CONNECT sockopt is set and
63619f6d3f3SWei Wang * we already have a valid cookie for this socket.
63719f6d3f3SWei Wang * In this case, user can call write() after connect().
63819f6d3f3SWei Wang * write() will invoke tcp_sendmsg_fastopen() which calls
63919f6d3f3SWei Wang * __inet_stream_connect().
64019f6d3f3SWei Wang */
64119f6d3f3SWei Wang if (uaddr) {
6426503d961SChangli Gao if (addr_len < sizeof(uaddr->sa_family))
6436503d961SChangli Gao return -EINVAL;
6446503d961SChangli Gao
6451da177e4SLinus Torvalds if (uaddr->sa_family == AF_UNSPEC) {
646419ce133SPaolo Abeni sk->sk_disconnects++;
6471da177e4SLinus Torvalds err = sk->sk_prot->disconnect(sk, flags);
6481da177e4SLinus Torvalds sock->state = err ? SS_DISCONNECTING : SS_UNCONNECTED;
6491da177e4SLinus Torvalds goto out;
6501da177e4SLinus Torvalds }
65119f6d3f3SWei Wang }
6521da177e4SLinus Torvalds
6531da177e4SLinus Torvalds switch (sock->state) {
6541da177e4SLinus Torvalds default:
6551da177e4SLinus Torvalds err = -EINVAL;
6561da177e4SLinus Torvalds goto out;
6571da177e4SLinus Torvalds case SS_CONNECTED:
6581da177e4SLinus Torvalds err = -EISCONN;
6591da177e4SLinus Torvalds goto out;
6601da177e4SLinus Torvalds case SS_CONNECTING:
66108e39c0dSEric Dumazet if (inet_test_bit(DEFER_CONNECT, sk))
6623979ad7eSWilly Tarreau err = is_sendmsg ? -EINPROGRESS : -EISCONN;
66319f6d3f3SWei Wang else
6641da177e4SLinus Torvalds err = -EALREADY;
6651da177e4SLinus Torvalds /* Fall out of switch with err, set for this state */
6661da177e4SLinus Torvalds break;
6671da177e4SLinus Torvalds case SS_UNCONNECTED:
6681da177e4SLinus Torvalds err = -EISCONN;
6691da177e4SLinus Torvalds if (sk->sk_state != TCP_CLOSE)
6701da177e4SLinus Torvalds goto out;
6711da177e4SLinus Torvalds
672d74bad4eSAndrey Ignatov if (BPF_CGROUP_PRE_CONNECT_ENABLED(sk)) {
673d74bad4eSAndrey Ignatov err = sk->sk_prot->pre_connect(sk, uaddr, addr_len);
674d74bad4eSAndrey Ignatov if (err)
675d74bad4eSAndrey Ignatov goto out;
676d74bad4eSAndrey Ignatov }
677d74bad4eSAndrey Ignatov
6781da177e4SLinus Torvalds err = sk->sk_prot->connect(sk, uaddr, addr_len);
6791da177e4SLinus Torvalds if (err < 0)
6801da177e4SLinus Torvalds goto out;
6811da177e4SLinus Torvalds
6821da177e4SLinus Torvalds sock->state = SS_CONNECTING;
6831da177e4SLinus Torvalds
68408e39c0dSEric Dumazet if (!err && inet_test_bit(DEFER_CONNECT, sk))
68519f6d3f3SWei Wang goto out;
68619f6d3f3SWei Wang
6871da177e4SLinus Torvalds /* Just entered SS_CONNECTING state; the only
6881da177e4SLinus Torvalds * difference is that return value in non-blocking
6891da177e4SLinus Torvalds * case is EINPROGRESS, rather than EALREADY.
6901da177e4SLinus Torvalds */
6911da177e4SLinus Torvalds err = -EINPROGRESS;
6921da177e4SLinus Torvalds break;
6931da177e4SLinus Torvalds }
6941da177e4SLinus Torvalds
6951da177e4SLinus Torvalds timeo = sock_sndtimeo(sk, flags & O_NONBLOCK);
6961da177e4SLinus Torvalds
6971da177e4SLinus Torvalds if ((1 << sk->sk_state) & (TCPF_SYN_SENT | TCPF_SYN_RECV)) {
698783237e8SYuchung Cheng int writebias = (sk->sk_protocol == IPPROTO_TCP) &&
699783237e8SYuchung Cheng tcp_sk(sk)->fastopen_req &&
700783237e8SYuchung Cheng tcp_sk(sk)->fastopen_req->data ? 1 : 0;
701419ce133SPaolo Abeni int dis = sk->sk_disconnects;
702783237e8SYuchung Cheng
7031da177e4SLinus Torvalds /* Error code is set above */
704783237e8SYuchung Cheng if (!timeo || !inet_wait_for_connect(sk, timeo, writebias))
7051da177e4SLinus Torvalds goto out;
7061da177e4SLinus Torvalds
7071da177e4SLinus Torvalds err = sock_intr_errno(timeo);
7081da177e4SLinus Torvalds if (signal_pending(current))
7091da177e4SLinus Torvalds goto out;
710419ce133SPaolo Abeni
711419ce133SPaolo Abeni if (dis != sk->sk_disconnects) {
712419ce133SPaolo Abeni err = -EPIPE;
713419ce133SPaolo Abeni goto out;
714419ce133SPaolo Abeni }
7151da177e4SLinus Torvalds }
7161da177e4SLinus Torvalds
7171da177e4SLinus Torvalds /* Connection was closed by RST, timeout, ICMP error
7181da177e4SLinus Torvalds * or another process disconnected us.
7191da177e4SLinus Torvalds */
7201da177e4SLinus Torvalds if (sk->sk_state == TCP_CLOSE)
7211da177e4SLinus Torvalds goto sock_error;
7221da177e4SLinus Torvalds
7231da177e4SLinus Torvalds /* sk->sk_err may be not zero now, if RECVERR was ordered by user
7241da177e4SLinus Torvalds * and error was received after socket entered established state.
7251da177e4SLinus Torvalds * Hence, it is handled normally after connect() return successfully.
7261da177e4SLinus Torvalds */
7271da177e4SLinus Torvalds
7281da177e4SLinus Torvalds sock->state = SS_CONNECTED;
7291da177e4SLinus Torvalds err = 0;
7301da177e4SLinus Torvalds out:
7311da177e4SLinus Torvalds return err;
7321da177e4SLinus Torvalds
7331da177e4SLinus Torvalds sock_error:
7341da177e4SLinus Torvalds err = sock_error(sk) ? : -ECONNABORTED;
7351da177e4SLinus Torvalds sock->state = SS_UNCONNECTED;
736419ce133SPaolo Abeni sk->sk_disconnects++;
7371da177e4SLinus Torvalds if (sk->sk_prot->disconnect(sk, flags))
7381da177e4SLinus Torvalds sock->state = SS_DISCONNECTING;
7391da177e4SLinus Torvalds goto out;
7401da177e4SLinus Torvalds }
741cf60af03SYuchung Cheng EXPORT_SYMBOL(__inet_stream_connect);
742cf60af03SYuchung Cheng
inet_stream_connect(struct socket * sock,struct sockaddr * uaddr,int addr_len,int flags)743cf60af03SYuchung Cheng int inet_stream_connect(struct socket *sock, struct sockaddr *uaddr,
744cf60af03SYuchung Cheng int addr_len, int flags)
745cf60af03SYuchung Cheng {
746cf60af03SYuchung Cheng int err;
747cf60af03SYuchung Cheng
748cf60af03SYuchung Cheng lock_sock(sock->sk);
7493979ad7eSWilly Tarreau err = __inet_stream_connect(sock, uaddr, addr_len, flags, 0);
750cf60af03SYuchung Cheng release_sock(sock->sk);
751cf60af03SYuchung Cheng return err;
752cf60af03SYuchung Cheng }
7533d1427f8SEric Dumazet EXPORT_SYMBOL(inet_stream_connect);
7541da177e4SLinus Torvalds
__inet_accept(struct socket * sock,struct socket * newsock,struct sock * newsk)755711bdd51SPaolo Abeni void __inet_accept(struct socket *sock, struct socket *newsock, struct sock *newsk)
756711bdd51SPaolo Abeni {
757711bdd51SPaolo Abeni sock_rps_record_flow(newsk);
758711bdd51SPaolo Abeni WARN_ON(!((1 << newsk->sk_state) &
759711bdd51SPaolo Abeni (TCPF_ESTABLISHED | TCPF_SYN_RECV |
76087bdc9f6SPaolo Abeni TCPF_FIN_WAIT1 | TCPF_FIN_WAIT2 |
76187bdc9f6SPaolo Abeni TCPF_CLOSING | TCPF_CLOSE_WAIT |
76287bdc9f6SPaolo Abeni TCPF_CLOSE)));
763711bdd51SPaolo Abeni
764711bdd51SPaolo Abeni if (test_bit(SOCK_SUPPORT_ZC, &sock->flags))
765711bdd51SPaolo Abeni set_bit(SOCK_SUPPORT_ZC, &newsock->flags);
766711bdd51SPaolo Abeni sock_graft(newsk, newsock);
767711bdd51SPaolo Abeni
768711bdd51SPaolo Abeni newsock->state = SS_CONNECTED;
769711bdd51SPaolo Abeni }
770711bdd51SPaolo Abeni
7711da177e4SLinus Torvalds /*
7721da177e4SLinus Torvalds * Accept a pending connection. The TCP layer now gives BSD semantics.
7731da177e4SLinus Torvalds */
7741da177e4SLinus Torvalds
inet_accept(struct socket * sock,struct socket * newsock,int flags,bool kern)775cdfbabfbSDavid Howells int inet_accept(struct socket *sock, struct socket *newsock, int flags,
776cdfbabfbSDavid Howells bool kern)
7771da177e4SLinus Torvalds {
778364f997bSKuniyuki Iwashima struct sock *sk1 = sock->sk, *sk2;
7791da177e4SLinus Torvalds int err = -EINVAL;
7801da177e4SLinus Torvalds
781364f997bSKuniyuki Iwashima /* IPV6_ADDRFORM can change sk->sk_prot under us. */
782364f997bSKuniyuki Iwashima sk2 = READ_ONCE(sk1->sk_prot)->accept(sk1, flags, &err, kern);
7831da177e4SLinus Torvalds if (!sk2)
784711bdd51SPaolo Abeni return err;
7851da177e4SLinus Torvalds
7861da177e4SLinus Torvalds lock_sock(sk2);
787711bdd51SPaolo Abeni __inet_accept(sock, newsock, sk2);
7881da177e4SLinus Torvalds release_sock(sk2);
789711bdd51SPaolo Abeni return 0;
7901da177e4SLinus Torvalds }
7913d1427f8SEric Dumazet EXPORT_SYMBOL(inet_accept);
7921da177e4SLinus Torvalds
7931da177e4SLinus Torvalds /*
7941da177e4SLinus Torvalds * This does both peername and sockname.
7951da177e4SLinus Torvalds */
inet_getname(struct socket * sock,struct sockaddr * uaddr,int peer)7961da177e4SLinus Torvalds int inet_getname(struct socket *sock, struct sockaddr *uaddr,
7979b2c45d4SDenys Vlasenko int peer)
7981da177e4SLinus Torvalds {
7991da177e4SLinus Torvalds struct sock *sk = sock->sk;
8001da177e4SLinus Torvalds struct inet_sock *inet = inet_sk(sk);
80138bfd8f5SCyrill Gorcunov DECLARE_SOCKADDR(struct sockaddr_in *, sin, uaddr);
8026d71331eSDaan De Meyer int sin_addr_len = sizeof(*sin);
8031da177e4SLinus Torvalds
8041da177e4SLinus Torvalds sin->sin_family = AF_INET;
8059dfc685eSEric Dumazet lock_sock(sk);
8061da177e4SLinus Torvalds if (peer) {
807c720c7e8SEric Dumazet if (!inet->inet_dport ||
8081da177e4SLinus Torvalds (((1 << sk->sk_state) & (TCPF_CLOSE | TCPF_SYN_SENT)) &&
8099dfc685eSEric Dumazet peer == 1)) {
8109dfc685eSEric Dumazet release_sock(sk);
8111da177e4SLinus Torvalds return -ENOTCONN;
8129dfc685eSEric Dumazet }
813c720c7e8SEric Dumazet sin->sin_port = inet->inet_dport;
814c720c7e8SEric Dumazet sin->sin_addr.s_addr = inet->inet_daddr;
8156d71331eSDaan De Meyer BPF_CGROUP_RUN_SA_PROG(sk, (struct sockaddr *)sin, &sin_addr_len,
8169dfc685eSEric Dumazet CGROUP_INET4_GETPEERNAME);
8171da177e4SLinus Torvalds } else {
818c720c7e8SEric Dumazet __be32 addr = inet->inet_rcv_saddr;
8191da177e4SLinus Torvalds if (!addr)
820c720c7e8SEric Dumazet addr = inet->inet_saddr;
821c720c7e8SEric Dumazet sin->sin_port = inet->inet_sport;
8221da177e4SLinus Torvalds sin->sin_addr.s_addr = addr;
8236d71331eSDaan De Meyer BPF_CGROUP_RUN_SA_PROG(sk, (struct sockaddr *)sin, &sin_addr_len,
8249dfc685eSEric Dumazet CGROUP_INET4_GETSOCKNAME);
825a9ed15daSStanislav Fomichev }
8269dfc685eSEric Dumazet release_sock(sk);
8271da177e4SLinus Torvalds memset(sin->sin_zero, 0, sizeof(sin->sin_zero));
8284690558eSMartin KaFai Lau return sin_addr_len;
8291da177e4SLinus Torvalds }
8303d1427f8SEric Dumazet EXPORT_SYMBOL(inet_getname);
8311da177e4SLinus Torvalds
inet_send_prepare(struct sock * sk)832e4730936SPaolo Abeni int inet_send_prepare(struct sock *sk)
8331da177e4SLinus Torvalds {
834c58dc01bSDavid S. Miller sock_rps_record_flow(sk);
835fec5e652STom Herbert
8361da177e4SLinus Torvalds /* We may need to bind the socket. */
837dcd01eeaSEric Dumazet if (data_race(!inet_sk(sk)->inet_num) && !sk->sk_prot->no_autobind &&
8387ba42910SChangli Gao inet_autobind(sk))
8391da177e4SLinus Torvalds return -EAGAIN;
8401da177e4SLinus Torvalds
841e4730936SPaolo Abeni return 0;
842e4730936SPaolo Abeni }
843e4730936SPaolo Abeni EXPORT_SYMBOL_GPL(inet_send_prepare);
844e4730936SPaolo Abeni
inet_sendmsg(struct socket * sock,struct msghdr * msg,size_t size)845e4730936SPaolo Abeni int inet_sendmsg(struct socket *sock, struct msghdr *msg, size_t size)
846e4730936SPaolo Abeni {
847e4730936SPaolo Abeni struct sock *sk = sock->sk;
848e4730936SPaolo Abeni
849e4730936SPaolo Abeni if (unlikely(inet_send_prepare(sk)))
850e4730936SPaolo Abeni return -EAGAIN;
851e4730936SPaolo Abeni
8526f24080eSPaolo Abeni return INDIRECT_CALL_2(sk->sk_prot->sendmsg, tcp_sendmsg, udp_sendmsg,
8536f24080eSPaolo Abeni sk, msg, size);
8541da177e4SLinus Torvalds }
8553d1427f8SEric Dumazet EXPORT_SYMBOL(inet_sendmsg);
8561da177e4SLinus Torvalds
inet_splice_eof(struct socket * sock)8571d7e4538SDavid Howells void inet_splice_eof(struct socket *sock)
8581d7e4538SDavid Howells {
8591d7e4538SDavid Howells const struct proto *prot;
8601d7e4538SDavid Howells struct sock *sk = sock->sk;
8611d7e4538SDavid Howells
8621d7e4538SDavid Howells if (unlikely(inet_send_prepare(sk)))
8631d7e4538SDavid Howells return;
8641d7e4538SDavid Howells
8651d7e4538SDavid Howells /* IPV6_ADDRFORM can change sk->sk_prot under us. */
8661d7e4538SDavid Howells prot = READ_ONCE(sk->sk_prot);
8671d7e4538SDavid Howells if (prot->splice_eof)
8681d7e4538SDavid Howells prot->splice_eof(sock);
8691d7e4538SDavid Howells }
8701d7e4538SDavid Howells EXPORT_SYMBOL_GPL(inet_splice_eof);
8711d7e4538SDavid Howells
8726f24080eSPaolo Abeni INDIRECT_CALLABLE_DECLARE(int udp_recvmsg(struct sock *, struct msghdr *,
873ec095263SOliver Hartkopp size_t, int, int *));
inet_recvmsg(struct socket * sock,struct msghdr * msg,size_t size,int flags)8741b784140SYing Xue int inet_recvmsg(struct socket *sock, struct msghdr *msg, size_t size,
8751b784140SYing Xue int flags)
876fec5e652STom Herbert {
877fec5e652STom Herbert struct sock *sk = sock->sk;
878fec5e652STom Herbert int addr_len = 0;
879fec5e652STom Herbert int err;
880fec5e652STom Herbert
881e3f2c4a3SSoheil Hassas Yeganeh if (likely(!(flags & MSG_ERRQUEUE)))
882c58dc01bSDavid S. Miller sock_rps_record_flow(sk);
883fec5e652STom Herbert
8846f24080eSPaolo Abeni err = INDIRECT_CALL_2(sk->sk_prot->recvmsg, tcp_recvmsg, udp_recvmsg,
885ec095263SOliver Hartkopp sk, msg, size, flags, &addr_len);
886fec5e652STom Herbert if (err >= 0)
887fec5e652STom Herbert msg->msg_namelen = addr_len;
888fec5e652STom Herbert return err;
889fec5e652STom Herbert }
890fec5e652STom Herbert EXPORT_SYMBOL(inet_recvmsg);
8911da177e4SLinus Torvalds
inet_shutdown(struct socket * sock,int how)8921da177e4SLinus Torvalds int inet_shutdown(struct socket *sock, int how)
8931da177e4SLinus Torvalds {
8941da177e4SLinus Torvalds struct sock *sk = sock->sk;
8951da177e4SLinus Torvalds int err = 0;
8961da177e4SLinus Torvalds
8971da177e4SLinus Torvalds /* This should really check to make sure
8981da177e4SLinus Torvalds * the socket is a TCP socket. (WHY AC...)
8991da177e4SLinus Torvalds */
9001da177e4SLinus Torvalds how++; /* maps 0->1 has the advantage of making bit 1 rcvs and
9011da177e4SLinus Torvalds 1->2 bit 2 snds.
9021da177e4SLinus Torvalds 2->3 */
9031da177e4SLinus Torvalds if ((how & ~SHUTDOWN_MASK) || !how) /* MAXINT->0 */
9041da177e4SLinus Torvalds return -EINVAL;
9051da177e4SLinus Torvalds
9061da177e4SLinus Torvalds lock_sock(sk);
9071da177e4SLinus Torvalds if (sock->state == SS_CONNECTING) {
9081da177e4SLinus Torvalds if ((1 << sk->sk_state) &
9091da177e4SLinus Torvalds (TCPF_SYN_SENT | TCPF_SYN_RECV | TCPF_CLOSE))
9101da177e4SLinus Torvalds sock->state = SS_DISCONNECTING;
9111da177e4SLinus Torvalds else
9121da177e4SLinus Torvalds sock->state = SS_CONNECTED;
9131da177e4SLinus Torvalds }
9141da177e4SLinus Torvalds
9151da177e4SLinus Torvalds switch (sk->sk_state) {
9161da177e4SLinus Torvalds case TCP_CLOSE:
9171da177e4SLinus Torvalds err = -ENOTCONN;
9181da177e4SLinus Torvalds /* Hack to wake up other listeners, who can poll for
919a9a08845SLinus Torvalds EPOLLHUP, even on eg. unconnected UDP sockets -- RR */
920a8eceea8SJoe Perches fallthrough;
9211da177e4SLinus Torvalds default:
922e14cadfdSEric Dumazet WRITE_ONCE(sk->sk_shutdown, sk->sk_shutdown | how);
9231da177e4SLinus Torvalds if (sk->sk_prot->shutdown)
9241da177e4SLinus Torvalds sk->sk_prot->shutdown(sk, how);
9251da177e4SLinus Torvalds break;
9261da177e4SLinus Torvalds
9271da177e4SLinus Torvalds /* Remaining two branches are temporary solution for missing
9281da177e4SLinus Torvalds * close() in multithreaded environment. It is _not_ a good idea,
9291da177e4SLinus Torvalds * but we have no choice until close() is repaired at VFS level.
9301da177e4SLinus Torvalds */
9311da177e4SLinus Torvalds case TCP_LISTEN:
9321da177e4SLinus Torvalds if (!(how & RCV_SHUTDOWN))
9331da177e4SLinus Torvalds break;
934a8eceea8SJoe Perches fallthrough;
9351da177e4SLinus Torvalds case TCP_SYN_SENT:
9361da177e4SLinus Torvalds err = sk->sk_prot->disconnect(sk, O_NONBLOCK);
9371da177e4SLinus Torvalds sock->state = err ? SS_DISCONNECTING : SS_UNCONNECTED;
9381da177e4SLinus Torvalds break;
9391da177e4SLinus Torvalds }
9401da177e4SLinus Torvalds
9411da177e4SLinus Torvalds /* Wake up anyone sleeping in poll. */
9421da177e4SLinus Torvalds sk->sk_state_change(sk);
9431da177e4SLinus Torvalds release_sock(sk);
9441da177e4SLinus Torvalds return err;
9451da177e4SLinus Torvalds }
9463d1427f8SEric Dumazet EXPORT_SYMBOL(inet_shutdown);
9471da177e4SLinus Torvalds
9481da177e4SLinus Torvalds /*
9491da177e4SLinus Torvalds * ioctl() calls you can issue on an INET socket. Most of these are
9501da177e4SLinus Torvalds * device configuration and stuff and very rarely used. Some ioctls
9511da177e4SLinus Torvalds * pass on to the socket itself.
9521da177e4SLinus Torvalds *
9531da177e4SLinus Torvalds * NOTE: I like the idea of a module for the config stuff. ie ifconfig
9541da177e4SLinus Torvalds * loads the devconfigure module does its configuring and unloads it.
9551da177e4SLinus Torvalds * There's a good 20K of config code hanging around the kernel.
9561da177e4SLinus Torvalds */
9571da177e4SLinus Torvalds
inet_ioctl(struct socket * sock,unsigned int cmd,unsigned long arg)9581da177e4SLinus Torvalds int inet_ioctl(struct socket *sock, unsigned int cmd, unsigned long arg)
9591da177e4SLinus Torvalds {
9601da177e4SLinus Torvalds struct sock *sk = sock->sk;
9611da177e4SLinus Torvalds int err = 0;
9623b1e0a65SYOSHIFUJI Hideaki struct net *net = sock_net(sk);
96303aef17bSAl Viro void __user *p = (void __user *)arg;
96403aef17bSAl Viro struct ifreq ifr;
965ca25c300SAl Viro struct rtentry rt;
9661da177e4SLinus Torvalds
9671da177e4SLinus Torvalds switch (cmd) {
9681da177e4SLinus Torvalds case SIOCADDRT:
9691da177e4SLinus Torvalds case SIOCDELRT:
970ca25c300SAl Viro if (copy_from_user(&rt, p, sizeof(struct rtentry)))
971ca25c300SAl Viro return -EFAULT;
972ca25c300SAl Viro err = ip_rt_ioctl(net, cmd, &rt);
973ca25c300SAl Viro break;
9741da177e4SLinus Torvalds case SIOCRTMSG:
975ca25c300SAl Viro err = -EINVAL;
9761da177e4SLinus Torvalds break;
9771da177e4SLinus Torvalds case SIOCDARP:
9781da177e4SLinus Torvalds case SIOCGARP:
9791da177e4SLinus Torvalds case SIOCSARP:
980e5b13cb1SDenis V. Lunev err = arp_ioctl(net, cmd, (void __user *)arg);
9811da177e4SLinus Torvalds break;
9821da177e4SLinus Torvalds case SIOCGIFADDR:
9831da177e4SLinus Torvalds case SIOCGIFBRDADDR:
9841da177e4SLinus Torvalds case SIOCGIFNETMASK:
9851da177e4SLinus Torvalds case SIOCGIFDSTADDR:
98603aef17bSAl Viro case SIOCGIFPFLAGS:
98729c49648SArnd Bergmann if (get_user_ifreq(&ifr, NULL, p))
98803aef17bSAl Viro return -EFAULT;
98903aef17bSAl Viro err = devinet_ioctl(net, cmd, &ifr);
99029c49648SArnd Bergmann if (!err && put_user_ifreq(&ifr, p))
99103aef17bSAl Viro err = -EFAULT;
99203aef17bSAl Viro break;
99303aef17bSAl Viro
99403aef17bSAl Viro case SIOCSIFADDR:
99503aef17bSAl Viro case SIOCSIFBRDADDR:
99603aef17bSAl Viro case SIOCSIFNETMASK:
9971da177e4SLinus Torvalds case SIOCSIFDSTADDR:
9981da177e4SLinus Torvalds case SIOCSIFPFLAGS:
9991da177e4SLinus Torvalds case SIOCSIFFLAGS:
100029c49648SArnd Bergmann if (get_user_ifreq(&ifr, NULL, p))
100103aef17bSAl Viro return -EFAULT;
100203aef17bSAl Viro err = devinet_ioctl(net, cmd, &ifr);
10031da177e4SLinus Torvalds break;
10041da177e4SLinus Torvalds default:
1005b5e5fa5eSChristoph Hellwig if (sk->sk_prot->ioctl)
1006e1d001faSBreno Leitao err = sk_ioctl(sk, cmd, (void __user *)arg);
1007b5e5fa5eSChristoph Hellwig else
1008b5e5fa5eSChristoph Hellwig err = -ENOIOCTLCMD;
10091da177e4SLinus Torvalds break;
10101da177e4SLinus Torvalds }
10111da177e4SLinus Torvalds return err;
10121da177e4SLinus Torvalds }
10133d1427f8SEric Dumazet EXPORT_SYMBOL(inet_ioctl);
10141da177e4SLinus Torvalds
1015709b46e8SEric W. Biederman #ifdef CONFIG_COMPAT
inet_compat_routing_ioctl(struct sock * sk,unsigned int cmd,struct compat_rtentry __user * ur)1016dc13c876SChristoph Hellwig static int inet_compat_routing_ioctl(struct sock *sk, unsigned int cmd,
1017dc13c876SChristoph Hellwig struct compat_rtentry __user *ur)
1018dc13c876SChristoph Hellwig {
1019dc13c876SChristoph Hellwig compat_uptr_t rtdev;
1020dc13c876SChristoph Hellwig struct rtentry rt;
1021dc13c876SChristoph Hellwig
1022dc13c876SChristoph Hellwig if (copy_from_user(&rt.rt_dst, &ur->rt_dst,
1023dc13c876SChristoph Hellwig 3 * sizeof(struct sockaddr)) ||
1024dc13c876SChristoph Hellwig get_user(rt.rt_flags, &ur->rt_flags) ||
1025dc13c876SChristoph Hellwig get_user(rt.rt_metric, &ur->rt_metric) ||
1026dc13c876SChristoph Hellwig get_user(rt.rt_mtu, &ur->rt_mtu) ||
1027dc13c876SChristoph Hellwig get_user(rt.rt_window, &ur->rt_window) ||
1028dc13c876SChristoph Hellwig get_user(rt.rt_irtt, &ur->rt_irtt) ||
1029dc13c876SChristoph Hellwig get_user(rtdev, &ur->rt_dev))
1030dc13c876SChristoph Hellwig return -EFAULT;
1031dc13c876SChristoph Hellwig
1032dc13c876SChristoph Hellwig rt.rt_dev = compat_ptr(rtdev);
1033dc13c876SChristoph Hellwig return ip_rt_ioctl(sock_net(sk), cmd, &rt);
1034dc13c876SChristoph Hellwig }
1035dc13c876SChristoph Hellwig
inet_compat_ioctl(struct socket * sock,unsigned int cmd,unsigned long arg)1036686dc6b6SGerrit Renker static int inet_compat_ioctl(struct socket *sock, unsigned int cmd, unsigned long arg)
1037709b46e8SEric W. Biederman {
1038dc13c876SChristoph Hellwig void __user *argp = compat_ptr(arg);
1039709b46e8SEric W. Biederman struct sock *sk = sock->sk;
1040709b46e8SEric W. Biederman
1041dc13c876SChristoph Hellwig switch (cmd) {
1042dc13c876SChristoph Hellwig case SIOCADDRT:
1043dc13c876SChristoph Hellwig case SIOCDELRT:
1044dc13c876SChristoph Hellwig return inet_compat_routing_ioctl(sk, cmd, argp);
1045dc13c876SChristoph Hellwig default:
1046dc13c876SChristoph Hellwig if (!sk->sk_prot->compat_ioctl)
1047dc13c876SChristoph Hellwig return -ENOIOCTLCMD;
1048dc13c876SChristoph Hellwig return sk->sk_prot->compat_ioctl(sk, cmd, arg);
1049709b46e8SEric W. Biederman }
1050dc13c876SChristoph Hellwig }
1051dc13c876SChristoph Hellwig #endif /* CONFIG_COMPAT */
1052709b46e8SEric W. Biederman
105390ddc4f0SEric Dumazet const struct proto_ops inet_stream_ops = {
10541da177e4SLinus Torvalds .family = PF_INET,
10551da177e4SLinus Torvalds .owner = THIS_MODULE,
10561da177e4SLinus Torvalds .release = inet_release,
10571da177e4SLinus Torvalds .bind = inet_bind,
10581da177e4SLinus Torvalds .connect = inet_stream_connect,
10591da177e4SLinus Torvalds .socketpair = sock_no_socketpair,
10601da177e4SLinus Torvalds .accept = inet_accept,
10611da177e4SLinus Torvalds .getname = inet_getname,
1062a11e1d43SLinus Torvalds .poll = tcp_poll,
10631da177e4SLinus Torvalds .ioctl = inet_ioctl,
1064c7cbdbf2SArnd Bergmann .gettstamp = sock_gettstamp,
10651da177e4SLinus Torvalds .listen = inet_listen,
10661da177e4SLinus Torvalds .shutdown = inet_shutdown,
10671da177e4SLinus Torvalds .setsockopt = sock_common_setsockopt,
10681da177e4SLinus Torvalds .getsockopt = sock_common_getsockopt,
10697ba42910SChangli Gao .sendmsg = inet_sendmsg,
1070fec5e652STom Herbert .recvmsg = inet_recvmsg,
107105255b82SEric Dumazet #ifdef CONFIG_MMU
107293ab6cc6SEric Dumazet .mmap = tcp_mmap,
107305255b82SEric Dumazet #endif
10741d7e4538SDavid Howells .splice_eof = inet_splice_eof,
10759c55e01cSJens Axboe .splice_read = tcp_splice_read,
107632035585STom Herbert .read_sock = tcp_read_sock,
1077965b57b4SCong Wang .read_skb = tcp_read_skb,
1078306b13ebSTom Herbert .sendmsg_locked = tcp_sendmsg_locked,
107932035585STom Herbert .peek_len = tcp_peek_len,
10803fdadf7dSDmitry Mishin #ifdef CONFIG_COMPAT
1081709b46e8SEric W. Biederman .compat_ioctl = inet_compat_ioctl,
10823fdadf7dSDmitry Mishin #endif
1083d1361840SEric Dumazet .set_rcvlowat = tcp_set_rcvlowat,
10841da177e4SLinus Torvalds };
10853d1427f8SEric Dumazet EXPORT_SYMBOL(inet_stream_ops);
10861da177e4SLinus Torvalds
108790ddc4f0SEric Dumazet const struct proto_ops inet_dgram_ops = {
10881da177e4SLinus Torvalds .family = PF_INET,
10891da177e4SLinus Torvalds .owner = THIS_MODULE,
10901da177e4SLinus Torvalds .release = inet_release,
10911da177e4SLinus Torvalds .bind = inet_bind,
10921da177e4SLinus Torvalds .connect = inet_dgram_connect,
10931da177e4SLinus Torvalds .socketpair = sock_no_socketpair,
10941da177e4SLinus Torvalds .accept = sock_no_accept,
10951da177e4SLinus Torvalds .getname = inet_getname,
1096a11e1d43SLinus Torvalds .poll = udp_poll,
10971da177e4SLinus Torvalds .ioctl = inet_ioctl,
1098c7cbdbf2SArnd Bergmann .gettstamp = sock_gettstamp,
10991da177e4SLinus Torvalds .listen = sock_no_listen,
11001da177e4SLinus Torvalds .shutdown = inet_shutdown,
11011da177e4SLinus Torvalds .setsockopt = sock_common_setsockopt,
11021da177e4SLinus Torvalds .getsockopt = sock_common_getsockopt,
11031da177e4SLinus Torvalds .sendmsg = inet_sendmsg,
1104965b57b4SCong Wang .read_skb = udp_read_skb,
1105fec5e652STom Herbert .recvmsg = inet_recvmsg,
11061da177e4SLinus Torvalds .mmap = sock_no_mmap,
11071d7e4538SDavid Howells .splice_eof = inet_splice_eof,
1108627d2d6bSsamanthakumar .set_peek_off = sk_set_peek_off,
1109543d9cfeSArnaldo Carvalho de Melo #ifdef CONFIG_COMPAT
1110709b46e8SEric W. Biederman .compat_ioctl = inet_compat_ioctl,
1111543d9cfeSArnaldo Carvalho de Melo #endif
11121da177e4SLinus Torvalds };
11133d1427f8SEric Dumazet EXPORT_SYMBOL(inet_dgram_ops);
11141da177e4SLinus Torvalds
11151da177e4SLinus Torvalds /*
11161da177e4SLinus Torvalds * For SOCK_RAW sockets; should be the same as inet_dgram_ops but without
1117a11e1d43SLinus Torvalds * udp_poll
11181da177e4SLinus Torvalds */
111990ddc4f0SEric Dumazet static const struct proto_ops inet_sockraw_ops = {
11201da177e4SLinus Torvalds .family = PF_INET,
11211da177e4SLinus Torvalds .owner = THIS_MODULE,
11221da177e4SLinus Torvalds .release = inet_release,
11231da177e4SLinus Torvalds .bind = inet_bind,
11241da177e4SLinus Torvalds .connect = inet_dgram_connect,
11251da177e4SLinus Torvalds .socketpair = sock_no_socketpair,
11261da177e4SLinus Torvalds .accept = sock_no_accept,
11271da177e4SLinus Torvalds .getname = inet_getname,
1128a11e1d43SLinus Torvalds .poll = datagram_poll,
11291da177e4SLinus Torvalds .ioctl = inet_ioctl,
1130c7cbdbf2SArnd Bergmann .gettstamp = sock_gettstamp,
11311da177e4SLinus Torvalds .listen = sock_no_listen,
11321da177e4SLinus Torvalds .shutdown = inet_shutdown,
11331da177e4SLinus Torvalds .setsockopt = sock_common_setsockopt,
11341da177e4SLinus Torvalds .getsockopt = sock_common_getsockopt,
11351da177e4SLinus Torvalds .sendmsg = inet_sendmsg,
1136fec5e652STom Herbert .recvmsg = inet_recvmsg,
11371da177e4SLinus Torvalds .mmap = sock_no_mmap,
11381d7e4538SDavid Howells .splice_eof = inet_splice_eof,
1139543d9cfeSArnaldo Carvalho de Melo #ifdef CONFIG_COMPAT
1140709b46e8SEric W. Biederman .compat_ioctl = inet_compat_ioctl,
1141543d9cfeSArnaldo Carvalho de Melo #endif
11421da177e4SLinus Torvalds };
11431da177e4SLinus Torvalds
1144ec1b4cf7SStephen Hemminger static const struct net_proto_family inet_family_ops = {
11451da177e4SLinus Torvalds .family = PF_INET,
11461da177e4SLinus Torvalds .create = inet_create,
11471da177e4SLinus Torvalds .owner = THIS_MODULE,
11481da177e4SLinus Torvalds };
11491da177e4SLinus Torvalds
11501da177e4SLinus Torvalds /* Upon startup we insert all the elements in inetsw_array[] into
11511da177e4SLinus Torvalds * the linked list inetsw.
11521da177e4SLinus Torvalds */
11531da177e4SLinus Torvalds static struct inet_protosw inetsw_array[] =
11541da177e4SLinus Torvalds {
11551da177e4SLinus Torvalds {
11561da177e4SLinus Torvalds .type = SOCK_STREAM,
11571da177e4SLinus Torvalds .protocol = IPPROTO_TCP,
11581da177e4SLinus Torvalds .prot = &tcp_prot,
11591da177e4SLinus Torvalds .ops = &inet_stream_ops,
1160d83d8461SArnaldo Carvalho de Melo .flags = INET_PROTOSW_PERMANENT |
1161d83d8461SArnaldo Carvalho de Melo INET_PROTOSW_ICSK,
11621da177e4SLinus Torvalds },
11631da177e4SLinus Torvalds
11641da177e4SLinus Torvalds {
11651da177e4SLinus Torvalds .type = SOCK_DGRAM,
11661da177e4SLinus Torvalds .protocol = IPPROTO_UDP,
11671da177e4SLinus Torvalds .prot = &udp_prot,
11681da177e4SLinus Torvalds .ops = &inet_dgram_ops,
11691da177e4SLinus Torvalds .flags = INET_PROTOSW_PERMANENT,
11701da177e4SLinus Torvalds },
11711da177e4SLinus Torvalds
1172c319b4d7SVasiliy Kulikov {
1173c319b4d7SVasiliy Kulikov .type = SOCK_DGRAM,
1174c319b4d7SVasiliy Kulikov .protocol = IPPROTO_ICMP,
1175c319b4d7SVasiliy Kulikov .prot = &ping_prot,
117677d4b1d3SEric Dumazet .ops = &inet_sockraw_ops,
1177c319b4d7SVasiliy Kulikov .flags = INET_PROTOSW_REUSE,
1178c319b4d7SVasiliy Kulikov },
11791da177e4SLinus Torvalds
11801da177e4SLinus Torvalds {
11811da177e4SLinus Torvalds .type = SOCK_RAW,
11821da177e4SLinus Torvalds .protocol = IPPROTO_IP, /* wild card */
11831da177e4SLinus Torvalds .prot = &raw_prot,
11841da177e4SLinus Torvalds .ops = &inet_sockraw_ops,
11851da177e4SLinus Torvalds .flags = INET_PROTOSW_REUSE,
11861da177e4SLinus Torvalds }
11871da177e4SLinus Torvalds };
11881da177e4SLinus Torvalds
1189c40f6fffSDenis Cheng #define INETSW_ARRAY_LEN ARRAY_SIZE(inetsw_array)
11901da177e4SLinus Torvalds
inet_register_protosw(struct inet_protosw * p)11911da177e4SLinus Torvalds void inet_register_protosw(struct inet_protosw *p)
11921da177e4SLinus Torvalds {
11931da177e4SLinus Torvalds struct list_head *lh;
11941da177e4SLinus Torvalds struct inet_protosw *answer;
11951da177e4SLinus Torvalds int protocol = p->protocol;
11961da177e4SLinus Torvalds struct list_head *last_perm;
11971da177e4SLinus Torvalds
11981da177e4SLinus Torvalds spin_lock_bh(&inetsw_lock);
11991da177e4SLinus Torvalds
12001da177e4SLinus Torvalds if (p->type >= SOCK_MAX)
12011da177e4SLinus Torvalds goto out_illegal;
12021da177e4SLinus Torvalds
12031da177e4SLinus Torvalds /* If we are trying to override a permanent protocol, bail. */
12041da177e4SLinus Torvalds last_perm = &inetsw[p->type];
12051da177e4SLinus Torvalds list_for_each(lh, &inetsw[p->type]) {
12061da177e4SLinus Torvalds answer = list_entry(lh, struct inet_protosw, list);
12071da177e4SLinus Torvalds /* Check only the non-wild match. */
1208f6c53334SJunwei Zhang if ((INET_PROTOSW_PERMANENT & answer->flags) == 0)
12091da177e4SLinus Torvalds break;
1210f6c53334SJunwei Zhang if (protocol == answer->protocol)
1211f6c53334SJunwei Zhang goto out_permanent;
12121da177e4SLinus Torvalds last_perm = lh;
12131da177e4SLinus Torvalds }
12141da177e4SLinus Torvalds
12151da177e4SLinus Torvalds /* Add the new entry after the last permanent entry if any, so that
12161da177e4SLinus Torvalds * the new entry does not override a permanent entry when matched with
12171da177e4SLinus Torvalds * a wild-card protocol. But it is allowed to override any existing
12181da177e4SLinus Torvalds * non-permanent entry. This means that when we remove this entry, the
12191da177e4SLinus Torvalds * system automatically returns to the old behavior.
12201da177e4SLinus Torvalds */
12211da177e4SLinus Torvalds list_add_rcu(&p->list, last_perm);
12221da177e4SLinus Torvalds out:
12231da177e4SLinus Torvalds spin_unlock_bh(&inetsw_lock);
12241da177e4SLinus Torvalds
12251da177e4SLinus Torvalds return;
12261da177e4SLinus Torvalds
12271da177e4SLinus Torvalds out_permanent:
1228058bd4d2SJoe Perches pr_err("Attempt to override permanent protocol %d\n", protocol);
12291da177e4SLinus Torvalds goto out;
12301da177e4SLinus Torvalds
12311da177e4SLinus Torvalds out_illegal:
1232058bd4d2SJoe Perches pr_err("Ignoring attempt to register invalid socket type %d\n",
12331da177e4SLinus Torvalds p->type);
12341da177e4SLinus Torvalds goto out;
12351da177e4SLinus Torvalds }
12363d1427f8SEric Dumazet EXPORT_SYMBOL(inet_register_protosw);
12371da177e4SLinus Torvalds
inet_unregister_protosw(struct inet_protosw * p)12381da177e4SLinus Torvalds void inet_unregister_protosw(struct inet_protosw *p)
12391da177e4SLinus Torvalds {
12401da177e4SLinus Torvalds if (INET_PROTOSW_PERMANENT & p->flags) {
1241058bd4d2SJoe Perches pr_err("Attempt to unregister permanent protocol %d\n",
12421da177e4SLinus Torvalds p->protocol);
12431da177e4SLinus Torvalds } else {
12441da177e4SLinus Torvalds spin_lock_bh(&inetsw_lock);
12451da177e4SLinus Torvalds list_del_rcu(&p->list);
12461da177e4SLinus Torvalds spin_unlock_bh(&inetsw_lock);
12471da177e4SLinus Torvalds
12481da177e4SLinus Torvalds synchronize_net();
12491da177e4SLinus Torvalds }
12501da177e4SLinus Torvalds }
12513d1427f8SEric Dumazet EXPORT_SYMBOL(inet_unregister_protosw);
12521da177e4SLinus Torvalds
inet_sk_reselect_saddr(struct sock * sk)125332519f11SArnaldo Carvalho de Melo static int inet_sk_reselect_saddr(struct sock *sk)
125432519f11SArnaldo Carvalho de Melo {
125532519f11SArnaldo Carvalho de Melo struct inet_sock *inet = inet_sk(sk);
1256c720c7e8SEric Dumazet __be32 old_saddr = inet->inet_saddr;
1257c720c7e8SEric Dumazet __be32 daddr = inet->inet_daddr;
12586e869138SDavid S. Miller struct flowi4 *fl4;
1259b23dd4feSDavid S. Miller struct rtable *rt;
1260b23dd4feSDavid S. Miller __be32 new_saddr;
1261f6d8bd05SEric Dumazet struct ip_options_rcu *inet_opt;
126228044fc1SJoanne Koong int err;
126332519f11SArnaldo Carvalho de Melo
1264f6d8bd05SEric Dumazet inet_opt = rcu_dereference_protected(inet->inet_opt,
12651e1d04e6SHannes Frederic Sowa lockdep_sock_is_held(sk));
1266f6d8bd05SEric Dumazet if (inet_opt && inet_opt->opt.srr)
1267f6d8bd05SEric Dumazet daddr = inet_opt->opt.faddr;
126832519f11SArnaldo Carvalho de Melo
126932519f11SArnaldo Carvalho de Melo /* Query new route. */
12706e869138SDavid S. Miller fl4 = &inet->cork.fl.u.ip4;
127167e1e2f4SGuillaume Nault rt = ip_route_connect(fl4, daddr, 0, sk->sk_bound_dev_if,
127267e1e2f4SGuillaume Nault sk->sk_protocol, inet->inet_sport,
127367e1e2f4SGuillaume Nault inet->inet_dport, sk);
1274b23dd4feSDavid S. Miller if (IS_ERR(rt))
1275b23dd4feSDavid S. Miller return PTR_ERR(rt);
127632519f11SArnaldo Carvalho de Melo
12776e869138SDavid S. Miller new_saddr = fl4->saddr;
127832519f11SArnaldo Carvalho de Melo
127928044fc1SJoanne Koong if (new_saddr == old_saddr) {
128028044fc1SJoanne Koong sk_setup_caps(sk, &rt->dst);
128132519f11SArnaldo Carvalho de Melo return 0;
128228044fc1SJoanne Koong }
128328044fc1SJoanne Koong
12848c5dae4cSKuniyuki Iwashima err = inet_bhash2_update_saddr(sk, &new_saddr, AF_INET);
128528044fc1SJoanne Koong if (err) {
128628044fc1SJoanne Koong ip_rt_put(rt);
128728044fc1SJoanne Koong return err;
128828044fc1SJoanne Koong }
128928044fc1SJoanne Koong
129028044fc1SJoanne Koong sk_setup_caps(sk, &rt->dst);
129132519f11SArnaldo Carvalho de Melo
1292e49e4affSKuniyuki Iwashima if (READ_ONCE(sock_net(sk)->ipv4.sysctl_ip_dynaddr) > 1) {
1293058bd4d2SJoe Perches pr_info("%s(): shifting inet->saddr from %pI4 to %pI4\n",
1294673d57e7SHarvey Harrison __func__, &old_saddr, &new_saddr);
129532519f11SArnaldo Carvalho de Melo }
129632519f11SArnaldo Carvalho de Melo
129732519f11SArnaldo Carvalho de Melo /*
129832519f11SArnaldo Carvalho de Melo * XXX The only one ugly spot where we need to
129932519f11SArnaldo Carvalho de Melo * XXX really change the sockets identity after
130032519f11SArnaldo Carvalho de Melo * XXX it has entered the hashes. -DaveM
130132519f11SArnaldo Carvalho de Melo *
130232519f11SArnaldo Carvalho de Melo * Besides that, it does not check for connection
130332519f11SArnaldo Carvalho de Melo * uniqueness. Wait for troubles.
130432519f11SArnaldo Carvalho de Melo */
1305086c653fSCraig Gallek return __sk_prot_rehash(sk);
130632519f11SArnaldo Carvalho de Melo }
130732519f11SArnaldo Carvalho de Melo
inet_sk_rebuild_header(struct sock * sk)130832519f11SArnaldo Carvalho de Melo int inet_sk_rebuild_header(struct sock *sk)
130932519f11SArnaldo Carvalho de Melo {
131032519f11SArnaldo Carvalho de Melo struct inet_sock *inet = inet_sk(sk);
131132519f11SArnaldo Carvalho de Melo struct rtable *rt = (struct rtable *)__sk_dst_check(sk, 0);
13123ca3c68eSAl Viro __be32 daddr;
1313f6d8bd05SEric Dumazet struct ip_options_rcu *inet_opt;
13146e869138SDavid S. Miller struct flowi4 *fl4;
131532519f11SArnaldo Carvalho de Melo int err;
131632519f11SArnaldo Carvalho de Melo
131732519f11SArnaldo Carvalho de Melo /* Route is OK, nothing to do. */
131832519f11SArnaldo Carvalho de Melo if (rt)
131932519f11SArnaldo Carvalho de Melo return 0;
132032519f11SArnaldo Carvalho de Melo
132132519f11SArnaldo Carvalho de Melo /* Reroute. */
1322f6d8bd05SEric Dumazet rcu_read_lock();
1323f6d8bd05SEric Dumazet inet_opt = rcu_dereference(inet->inet_opt);
1324c720c7e8SEric Dumazet daddr = inet->inet_daddr;
1325f6d8bd05SEric Dumazet if (inet_opt && inet_opt->opt.srr)
1326f6d8bd05SEric Dumazet daddr = inet_opt->opt.faddr;
1327f6d8bd05SEric Dumazet rcu_read_unlock();
13286e869138SDavid S. Miller fl4 = &inet->cork.fl.u.ip4;
13296e869138SDavid S. Miller rt = ip_route_output_ports(sock_net(sk), fl4, sk, daddr, inet->inet_saddr,
133078fbfd8aSDavid S. Miller inet->inet_dport, inet->inet_sport,
133178fbfd8aSDavid S. Miller sk->sk_protocol, RT_CONN_FLAGS(sk),
133278fbfd8aSDavid S. Miller sk->sk_bound_dev_if);
1333b23dd4feSDavid S. Miller if (!IS_ERR(rt)) {
1334b23dd4feSDavid S. Miller err = 0;
1335d8d1f30bSChangli Gao sk_setup_caps(sk, &rt->dst);
1336b23dd4feSDavid S. Miller } else {
1337b23dd4feSDavid S. Miller err = PTR_ERR(rt);
1338b23dd4feSDavid S. Miller
133932519f11SArnaldo Carvalho de Melo /* Routing failed... */
134032519f11SArnaldo Carvalho de Melo sk->sk_route_caps = 0;
134132519f11SArnaldo Carvalho de Melo /*
134232519f11SArnaldo Carvalho de Melo * Other protocols have to map its equivalent state to TCP_SYN_SENT.
134332519f11SArnaldo Carvalho de Melo * DCCP maps its DCCP_REQUESTING state to TCP_SYN_SENT. -acme
134432519f11SArnaldo Carvalho de Melo */
1345e49e4affSKuniyuki Iwashima if (!READ_ONCE(sock_net(sk)->ipv4.sysctl_ip_dynaddr) ||
134632519f11SArnaldo Carvalho de Melo sk->sk_state != TCP_SYN_SENT ||
134732519f11SArnaldo Carvalho de Melo (sk->sk_userlocks & SOCK_BINDADDR_LOCK) ||
134832519f11SArnaldo Carvalho de Melo (err = inet_sk_reselect_saddr(sk)) != 0)
13492f2d9972SEric Dumazet WRITE_ONCE(sk->sk_err_soft, -err);
135032519f11SArnaldo Carvalho de Melo }
135132519f11SArnaldo Carvalho de Melo
135232519f11SArnaldo Carvalho de Melo return err;
135332519f11SArnaldo Carvalho de Melo }
135432519f11SArnaldo Carvalho de Melo EXPORT_SYMBOL(inet_sk_rebuild_header);
135532519f11SArnaldo Carvalho de Melo
inet_sk_set_state(struct sock * sk,int state)1356563e0bb0SYafang Shao void inet_sk_set_state(struct sock *sk, int state)
1357563e0bb0SYafang Shao {
1358563e0bb0SYafang Shao trace_inet_sock_set_state(sk, sk->sk_state, state);
1359563e0bb0SYafang Shao sk->sk_state = state;
1360563e0bb0SYafang Shao }
1361563e0bb0SYafang Shao EXPORT_SYMBOL(inet_sk_set_state);
1362563e0bb0SYafang Shao
inet_sk_state_store(struct sock * sk,int newstate)1363563e0bb0SYafang Shao void inet_sk_state_store(struct sock *sk, int newstate)
1364563e0bb0SYafang Shao {
1365563e0bb0SYafang Shao trace_inet_sock_set_state(sk, sk->sk_state, newstate);
1366563e0bb0SYafang Shao smp_store_release(&sk->sk_state, newstate);
1367563e0bb0SYafang Shao }
1368563e0bb0SYafang Shao
inet_gso_segment(struct sk_buff * skb,netdev_features_t features)1369b8921ca8STom Herbert struct sk_buff *inet_gso_segment(struct sk_buff *skb,
1370c8f44affSMichał Mirosław netdev_features_t features)
1371f4c50d99SHerbert Xu {
13720c19f846SWillem de Bruijn bool udpfrag = false, fixedid = false, gso_partial, encap;
1373f4c50d99SHerbert Xu struct sk_buff *segs = ERR_PTR(-EINVAL);
1374bca49f84SVlad Yasevich const struct net_offload *ops;
13750c19f846SWillem de Bruijn unsigned int offset = 0;
1376f9242b6bSDavid S. Miller struct iphdr *iph;
1377802ab55aSAlexander Duyck int proto, tot_len;
13783347c960SEric Dumazet int nhoff;
1379f4c50d99SHerbert Xu int ihl;
1380f4c50d99SHerbert Xu int id;
1381f4c50d99SHerbert Xu
13823347c960SEric Dumazet skb_reset_network_header(skb);
13833347c960SEric Dumazet nhoff = skb_network_header(skb) - skb_mac_header(skb);
1384bbcf467dSHerbert Xu if (unlikely(!pskb_may_pull(skb, sizeof(*iph))))
1385f4c50d99SHerbert Xu goto out;
1386f4c50d99SHerbert Xu
1387eddc9ec5SArnaldo Carvalho de Melo iph = ip_hdr(skb);
1388f4c50d99SHerbert Xu ihl = iph->ihl * 4;
1389f4c50d99SHerbert Xu if (ihl < sizeof(*iph))
1390f4c50d99SHerbert Xu goto out;
1391f4c50d99SHerbert Xu
139247d27aadSEric Dumazet id = ntohs(iph->id);
139347d27aadSEric Dumazet proto = iph->protocol;
139447d27aadSEric Dumazet
139547d27aadSEric Dumazet /* Warning: after this point, iph might be no longer valid */
1396bbcf467dSHerbert Xu if (unlikely(!pskb_may_pull(skb, ihl)))
1397f4c50d99SHerbert Xu goto out;
139847d27aadSEric Dumazet __skb_pull(skb, ihl);
1399f4c50d99SHerbert Xu
14008c3a897bSEric Dumazet encap = SKB_GSO_CB(skb)->encap_level > 0;
14018c3a897bSEric Dumazet if (encap)
14021e16aa3dSFlorian Westphal features &= skb->dev->hw_enc_features;
14033347c960SEric Dumazet SKB_GSO_CB(skb)->encap_level += ihl;
140473136267SPravin B Shelar
1405badff6d0SArnaldo Carvalho de Melo skb_reset_transport_header(skb);
140647d27aadSEric Dumazet
1407f4c50d99SHerbert Xu segs = ERR_PTR(-EPROTONOSUPPORT);
1408f4c50d99SHerbert Xu
1409cbc53e08SAlexander Duyck if (!skb->encapsulation || encap) {
14100c19f846SWillem de Bruijn udpfrag = !!(skb_shinfo(skb)->gso_type & SKB_GSO_UDP);
1411cbc53e08SAlexander Duyck fixedid = !!(skb_shinfo(skb)->gso_type & SKB_GSO_TCP_FIXEDID);
1412cbc53e08SAlexander Duyck
1413cbc53e08SAlexander Duyck /* fixed ID is invalid if DF bit is not set */
1414a5108878SArnaldo Carvalho de Melo if (fixedid && !(ip_hdr(skb)->frag_off & htons(IP_DF)))
1415cbc53e08SAlexander Duyck goto out;
1416cbc53e08SAlexander Duyck }
1417dcd60771SEric Dumazet
1418bca49f84SVlad Yasevich ops = rcu_dereference(inet_offloads[proto]);
1419cc20ccedSTao Liu if (likely(ops && ops->callbacks.gso_segment)) {
1420f191a1d1SVlad Yasevich segs = ops->callbacks.gso_segment(skb, features);
1421cc20ccedSTao Liu if (!segs)
1422cc20ccedSTao Liu skb->network_header = skb_mac_header(skb) + nhoff - skb->head;
1423cc20ccedSTao Liu }
1424f4c50d99SHerbert Xu
142550c3a487SYOSHIFUJI Hideaki / 吉藤英明 if (IS_ERR_OR_NULL(segs))
1426f4c50d99SHerbert Xu goto out;
1427f4c50d99SHerbert Xu
142807b26c94SSteffen Klassert gso_partial = !!(skb_shinfo(segs)->gso_type & SKB_GSO_PARTIAL);
142907b26c94SSteffen Klassert
1430f4c50d99SHerbert Xu skb = segs;
1431f4c50d99SHerbert Xu do {
14323347c960SEric Dumazet iph = (struct iphdr *)(skb_mac_header(skb) + nhoff);
14330c19f846SWillem de Bruijn if (udpfrag) {
14340c19f846SWillem de Bruijn iph->frag_off = htons(offset >> 3);
14350c19f846SWillem de Bruijn if (skb->next)
14360c19f846SWillem de Bruijn iph->frag_off |= htons(IP_MF);
14370c19f846SWillem de Bruijn offset += skb->len - nhoff - ihl;
14380c19f846SWillem de Bruijn tot_len = skb->len - nhoff;
14390c19f846SWillem de Bruijn } else if (skb_is_gso(skb)) {
1440802ab55aSAlexander Duyck if (!fixedid) {
1441802ab55aSAlexander Duyck iph->id = htons(id);
1442802ab55aSAlexander Duyck id += skb_shinfo(skb)->gso_segs;
1443490ab081SPravin B Shelar }
144407b26c94SSteffen Klassert
144507b26c94SSteffen Klassert if (gso_partial)
1446802ab55aSAlexander Duyck tot_len = skb_shinfo(skb)->gso_size +
1447802ab55aSAlexander Duyck SKB_GSO_CB(skb)->data_offset +
1448802ab55aSAlexander Duyck skb->head - (unsigned char *)iph;
144907b26c94SSteffen Klassert else
145007b26c94SSteffen Klassert tot_len = skb->len - nhoff;
1451802ab55aSAlexander Duyck } else {
1452802ab55aSAlexander Duyck if (!fixedid)
1453802ab55aSAlexander Duyck iph->id = htons(id++);
1454802ab55aSAlexander Duyck tot_len = skb->len - nhoff;
1455802ab55aSAlexander Duyck }
1456802ab55aSAlexander Duyck iph->tot_len = htons(tot_len);
145747d27aadSEric Dumazet ip_send_check(iph);
14588c3a897bSEric Dumazet if (encap)
14593347c960SEric Dumazet skb_reset_inner_headers(skb);
14603347c960SEric Dumazet skb->network_header = (u8 *)iph - skb->head;
1461c56cae23SToke Høiland-Jørgensen skb_reset_mac_len(skb);
1462f4c50d99SHerbert Xu } while ((skb = skb->next));
1463f4c50d99SHerbert Xu
1464f4c50d99SHerbert Xu out:
1465f4c50d99SHerbert Xu return segs;
1466f4c50d99SHerbert Xu }
1467f4c50d99SHerbert Xu
ipip_gso_segment(struct sk_buff * skb,netdev_features_t features)1468418e897eSWillem de Bruijn static struct sk_buff *ipip_gso_segment(struct sk_buff *skb,
1469418e897eSWillem de Bruijn netdev_features_t features)
1470418e897eSWillem de Bruijn {
1471418e897eSWillem de Bruijn if (!(skb_shinfo(skb)->gso_type & SKB_GSO_IPXIP4))
1472418e897eSWillem de Bruijn return ERR_PTR(-EINVAL);
1473418e897eSWillem de Bruijn
1474418e897eSWillem de Bruijn return inet_gso_segment(skb, features);
1475418e897eSWillem de Bruijn }
1476418e897eSWillem de Bruijn
inet_gro_receive(struct list_head * head,struct sk_buff * skb)1477d4546c25SDavid Miller struct sk_buff *inet_gro_receive(struct list_head *head, struct sk_buff *skb)
147873cc19f1SHerbert Xu {
1479bca49f84SVlad Yasevich const struct net_offload *ops;
1480d4546c25SDavid Miller struct sk_buff *pp = NULL;
1481b71d1d42SEric Dumazet const struct iphdr *iph;
1482d4546c25SDavid Miller struct sk_buff *p;
1483a5b1cf28SHerbert Xu unsigned int hlen;
1484a5b1cf28SHerbert Xu unsigned int off;
14851075f3f6SHerbert Xu unsigned int id;
148673cc19f1SHerbert Xu int flush = 1;
148773cc19f1SHerbert Xu int proto;
148873cc19f1SHerbert Xu
1489a5b1cf28SHerbert Xu off = skb_gro_offset(skb);
1490a5b1cf28SHerbert Xu hlen = off + sizeof(*iph);
149135ffb665SRichard Gobert iph = skb_gro_header(skb, hlen, off);
149286911732SHerbert Xu if (unlikely(!iph))
149373cc19f1SHerbert Xu goto out;
149473cc19f1SHerbert Xu
1495f9242b6bSDavid S. Miller proto = iph->protocol;
149673cc19f1SHerbert Xu
1497bca49f84SVlad Yasevich ops = rcu_dereference(inet_offloads[proto]);
1498f191a1d1SVlad Yasevich if (!ops || !ops->callbacks.gro_receive)
1499fc1ca334SEric Dumazet goto out;
150073cc19f1SHerbert Xu
1501a5ad24beSHerbert Xu if (*(u8 *)iph != 0x45)
1502fc1ca334SEric Dumazet goto out;
150373cc19f1SHerbert Xu
15049b83e031SSteffen Klassert if (ip_is_fragment(iph))
1505fc1ca334SEric Dumazet goto out;
15069b83e031SSteffen Klassert
1507a9e050f4SEric Dumazet if (unlikely(ip_fast_csum((u8 *)iph, 5)))
1508fc1ca334SEric Dumazet goto out;
150973cc19f1SHerbert Xu
1510b1a78b9bSXin Long NAPI_GRO_CB(skb)->proto = proto;
15110eae88f3SEric Dumazet id = ntohl(*(__be32 *)&iph->id);
1512db8caf3dSEric Dumazet flush = (u16)((ntohl(*(__be32 *)iph) ^ skb_gro_len(skb)) | (id & ~IP_DF));
15131075f3f6SHerbert Xu id >>= 16;
151473cc19f1SHerbert Xu
1515d4546c25SDavid Miller list_for_each_entry(p, head, list) {
151673cc19f1SHerbert Xu struct iphdr *iph2;
15171530545eSAlexander Duyck u16 flush_id;
151873cc19f1SHerbert Xu
151973cc19f1SHerbert Xu if (!NAPI_GRO_CB(p)->same_flow)
152073cc19f1SHerbert Xu continue;
152173cc19f1SHerbert Xu
1522299603e8SJerry Chu iph2 = (struct iphdr *)(p->data + off);
1523299603e8SJerry Chu /* The above works because, with the exception of the top
1524299603e8SJerry Chu * (inner most) layer, we only aggregate pkts with the same
1525299603e8SJerry Chu * hdr length so all the hdrs we'll need to verify will start
1526299603e8SJerry Chu * at the same offset.
1527299603e8SJerry Chu */
1528a5ad24beSHerbert Xu if ((iph->protocol ^ iph2->protocol) |
15290eae88f3SEric Dumazet ((__force u32)iph->saddr ^ (__force u32)iph2->saddr) |
15300eae88f3SEric Dumazet ((__force u32)iph->daddr ^ (__force u32)iph2->daddr)) {
153173cc19f1SHerbert Xu NAPI_GRO_CB(p)->same_flow = 0;
153273cc19f1SHerbert Xu continue;
153373cc19f1SHerbert Xu }
153473cc19f1SHerbert Xu
153573cc19f1SHerbert Xu /* All fields must match except length and checksum. */
153673cc19f1SHerbert Xu NAPI_GRO_CB(p)->flush |=
1537a5ad24beSHerbert Xu (iph->ttl ^ iph2->ttl) |
1538a9e050f4SEric Dumazet (iph->tos ^ iph2->tos) |
1539bf5a755fSJerry Chu ((iph->frag_off ^ iph2->frag_off) & htons(IP_DF));
154073cc19f1SHerbert Xu
154173cc19f1SHerbert Xu NAPI_GRO_CB(p)->flush |= flush;
15421530545eSAlexander Duyck
15431530545eSAlexander Duyck /* We need to store of the IP ID check to be included later
15441530545eSAlexander Duyck * when we can verify that this packet does in fact belong
15451530545eSAlexander Duyck * to a given flow.
15461530545eSAlexander Duyck */
15471530545eSAlexander Duyck flush_id = (u16)(id - ntohs(iph2->id));
15481530545eSAlexander Duyck
15491530545eSAlexander Duyck /* This bit of code makes it much easier for us to identify
15501530545eSAlexander Duyck * the cases where we are doing atomic vs non-atomic IP ID
15511530545eSAlexander Duyck * checks. Specifically an atomic check can return IP ID
15521530545eSAlexander Duyck * values 0 - 0xFFFF, while a non-atomic check can only
15531530545eSAlexander Duyck * return 0 or 0xFFFF.
15541530545eSAlexander Duyck */
15551530545eSAlexander Duyck if (!NAPI_GRO_CB(p)->is_atomic ||
15561530545eSAlexander Duyck !(iph->frag_off & htons(IP_DF))) {
15571530545eSAlexander Duyck flush_id ^= NAPI_GRO_CB(p)->count;
15581530545eSAlexander Duyck flush_id = flush_id ? 0xFFFF : 0;
155973cc19f1SHerbert Xu }
156073cc19f1SHerbert Xu
15611530545eSAlexander Duyck /* If the previous IP ID value was based on an atomic
15621530545eSAlexander Duyck * datagram we can overwrite the value and ignore it.
15631530545eSAlexander Duyck */
15641530545eSAlexander Duyck if (NAPI_GRO_CB(skb)->is_atomic)
15651530545eSAlexander Duyck NAPI_GRO_CB(p)->flush_id = flush_id;
15661530545eSAlexander Duyck else
15671530545eSAlexander Duyck NAPI_GRO_CB(p)->flush_id |= flush_id;
15681530545eSAlexander Duyck }
15691530545eSAlexander Duyck
15701530545eSAlexander Duyck NAPI_GRO_CB(skb)->is_atomic = !!(iph->frag_off & htons(IP_DF));
157173cc19f1SHerbert Xu NAPI_GRO_CB(skb)->flush |= flush;
1572299603e8SJerry Chu skb_set_network_header(skb, off);
1573299603e8SJerry Chu /* The above will be needed by the transport layer if there is one
1574299603e8SJerry Chu * immediately following this IP hdr.
1575299603e8SJerry Chu */
1576af276a5aSRichard Gobert NAPI_GRO_CB(skb)->inner_network_offset = off;
1577299603e8SJerry Chu
15782c804d0fSEric Dumazet /* Note : No need to call skb_gro_postpull_rcsum() here,
15792c804d0fSEric Dumazet * as we already checked checksum over ipv4 header was 0
15802c804d0fSEric Dumazet */
158186911732SHerbert Xu skb_gro_pull(skb, sizeof(*iph));
158286911732SHerbert Xu skb_set_transport_header(skb, skb_gro_offset(skb));
158373cc19f1SHerbert Xu
1584028e0a47SPaolo Abeni pp = indirect_call_gro_receive(tcp4_gro_receive, udp4_gro_receive,
1585028e0a47SPaolo Abeni ops->callbacks.gro_receive, head, skb);
158673cc19f1SHerbert Xu
158773cc19f1SHerbert Xu out:
15885f114163SSteffen Klassert skb_gro_flush_final(skb, pp, flush);
158973cc19f1SHerbert Xu
159073cc19f1SHerbert Xu return pp;
159173cc19f1SHerbert Xu }
159273cc19f1SHerbert Xu
ipip_gro_receive(struct list_head * head,struct sk_buff * skb)1593d4546c25SDavid Miller static struct sk_buff *ipip_gro_receive(struct list_head *head,
1594fac8e0f5SJesse Gross struct sk_buff *skb)
1595fac8e0f5SJesse Gross {
1596fac8e0f5SJesse Gross if (NAPI_GRO_CB(skb)->encap_mark) {
1597fac8e0f5SJesse Gross NAPI_GRO_CB(skb)->flush = 1;
1598fac8e0f5SJesse Gross return NULL;
1599fac8e0f5SJesse Gross }
1600fac8e0f5SJesse Gross
1601fac8e0f5SJesse Gross NAPI_GRO_CB(skb)->encap_mark = 1;
1602fac8e0f5SJesse Gross
1603fac8e0f5SJesse Gross return inet_gro_receive(head, skb);
1604fac8e0f5SJesse Gross }
1605fac8e0f5SJesse Gross
1606822c8685SDeepa Dinamani #define SECONDS_PER_DAY 86400
1607822c8685SDeepa Dinamani
1608822c8685SDeepa Dinamani /* inet_current_timestamp - Return IP network timestamp
1609822c8685SDeepa Dinamani *
1610822c8685SDeepa Dinamani * Return milliseconds since midnight in network byte order.
1611822c8685SDeepa Dinamani */
inet_current_timestamp(void)1612822c8685SDeepa Dinamani __be32 inet_current_timestamp(void)
1613822c8685SDeepa Dinamani {
1614822c8685SDeepa Dinamani u32 secs;
1615822c8685SDeepa Dinamani u32 msecs;
1616822c8685SDeepa Dinamani struct timespec64 ts;
1617822c8685SDeepa Dinamani
1618822c8685SDeepa Dinamani ktime_get_real_ts64(&ts);
1619822c8685SDeepa Dinamani
1620822c8685SDeepa Dinamani /* Get secs since midnight. */
1621822c8685SDeepa Dinamani (void)div_u64_rem(ts.tv_sec, SECONDS_PER_DAY, &secs);
1622822c8685SDeepa Dinamani /* Convert to msecs. */
1623822c8685SDeepa Dinamani msecs = secs * MSEC_PER_SEC;
1624822c8685SDeepa Dinamani /* Convert nsec to msec. */
1625822c8685SDeepa Dinamani msecs += (u32)ts.tv_nsec / NSEC_PER_MSEC;
1626822c8685SDeepa Dinamani
1627822c8685SDeepa Dinamani /* Convert to network byte order. */
16283ba9d300SDeepa Dinamani return htonl(msecs);
1629822c8685SDeepa Dinamani }
1630822c8685SDeepa Dinamani EXPORT_SYMBOL(inet_current_timestamp);
1631822c8685SDeepa Dinamani
inet_recv_error(struct sock * sk,struct msghdr * msg,int len,int * addr_len)1632f4713a3dSWillem de Bruijn int inet_recv_error(struct sock *sk, struct msghdr *msg, int len, int *addr_len)
1633f4713a3dSWillem de Bruijn {
16344a5e31bdSEric Dumazet unsigned int family = READ_ONCE(sk->sk_family);
16354a5e31bdSEric Dumazet
16364a5e31bdSEric Dumazet if (family == AF_INET)
1637f4713a3dSWillem de Bruijn return ip_recv_error(sk, msg, len, addr_len);
1638f4713a3dSWillem de Bruijn #if IS_ENABLED(CONFIG_IPV6)
16394a5e31bdSEric Dumazet if (family == AF_INET6)
1640f4713a3dSWillem de Bruijn return pingv6_ops.ipv6_recv_error(sk, msg, len, addr_len);
1641f4713a3dSWillem de Bruijn #endif
1642f4713a3dSWillem de Bruijn return -EINVAL;
1643f4713a3dSWillem de Bruijn }
16449702e7fbSEric Dumazet EXPORT_SYMBOL(inet_recv_error);
1645f4713a3dSWillem de Bruijn
inet_gro_complete(struct sk_buff * skb,int nhoff)1646b8921ca8STom Herbert int inet_gro_complete(struct sk_buff *skb, int nhoff)
164773cc19f1SHerbert Xu {
1648299603e8SJerry Chu struct iphdr *iph = (struct iphdr *)(skb->data + nhoff);
1649bca49f84SVlad Yasevich const struct net_offload *ops;
1650b1a78b9bSXin Long __be16 totlen = iph->tot_len;
1651f9242b6bSDavid S. Miller int proto = iph->protocol;
1652f9242b6bSDavid S. Miller int err = -ENOSYS;
165373cc19f1SHerbert Xu
1654294acf1cSPaolo Abeni if (skb->encapsulation) {
1655294acf1cSPaolo Abeni skb_set_inner_protocol(skb, cpu_to_be16(ETH_P_IP));
1656c3caf119SJerry Chu skb_set_inner_network_header(skb, nhoff);
1657294acf1cSPaolo Abeni }
1658c3caf119SJerry Chu
1659b1a78b9bSXin Long iph_set_totlen(iph, skb->len - nhoff);
1660b1a78b9bSXin Long csum_replace2(&iph->check, totlen, iph->tot_len);
166173cc19f1SHerbert Xu
1662bca49f84SVlad Yasevich ops = rcu_dereference(inet_offloads[proto]);
1663f191a1d1SVlad Yasevich if (WARN_ON(!ops || !ops->callbacks.gro_complete))
1664627b94f7SEric Dumazet goto out;
166573cc19f1SHerbert Xu
1666299603e8SJerry Chu /* Only need to add sizeof(*iph) to get to the next hdr below
1667299603e8SJerry Chu * because any hdr with option will have been flushed in
1668299603e8SJerry Chu * inet_gro_receive().
1669299603e8SJerry Chu */
1670028e0a47SPaolo Abeni err = INDIRECT_CALL_2(ops->callbacks.gro_complete,
1671028e0a47SPaolo Abeni tcp4_gro_complete, udp4_gro_complete,
1672028e0a47SPaolo Abeni skb, nhoff + sizeof(*iph));
167373cc19f1SHerbert Xu
1674627b94f7SEric Dumazet out:
167573cc19f1SHerbert Xu return err;
167673cc19f1SHerbert Xu }
167773cc19f1SHerbert Xu
ipip_gro_complete(struct sk_buff * skb,int nhoff)1678b8cba75bSJesse Gross static int ipip_gro_complete(struct sk_buff *skb, int nhoff)
1679b8cba75bSJesse Gross {
1680b8cba75bSJesse Gross skb->encapsulation = 1;
16817e13318dSTom Herbert skb_shinfo(skb)->gso_type |= SKB_GSO_IPXIP4;
1682b8cba75bSJesse Gross return inet_gro_complete(skb, nhoff);
1683b8cba75bSJesse Gross }
1684b8cba75bSJesse Gross
inet_ctl_sock_create(struct sock ** sk,unsigned short family,unsigned short type,unsigned char protocol,struct net * net)1685eee4fe4dSDenis V. Lunev int inet_ctl_sock_create(struct sock **sk, unsigned short family,
16865677242fSDenis V. Lunev unsigned short type, unsigned char protocol,
16875677242fSDenis V. Lunev struct net *net)
16883d58b5faSDenis V. Lunev {
1689eee4fe4dSDenis V. Lunev struct socket *sock;
169026abe143SEric W. Biederman int rc = sock_create_kern(net, family, type, protocol, &sock);
16913d58b5faSDenis V. Lunev
16923d58b5faSDenis V. Lunev if (rc == 0) {
1693eee4fe4dSDenis V. Lunev *sk = sock->sk;
1694eee4fe4dSDenis V. Lunev (*sk)->sk_allocation = GFP_ATOMIC;
16951ac88557SEric Dumazet (*sk)->sk_use_task_frag = false;
16963d58b5faSDenis V. Lunev /*
16973d58b5faSDenis V. Lunev * Unhash it so that IP input processing does not even see it,
16983d58b5faSDenis V. Lunev * we do not wish this socket to see incoming packets.
16993d58b5faSDenis V. Lunev */
1700eee4fe4dSDenis V. Lunev (*sk)->sk_prot->unhash(*sk);
17013d58b5faSDenis V. Lunev }
17023d58b5faSDenis V. Lunev return rc;
17033d58b5faSDenis V. Lunev }
17043d58b5faSDenis V. Lunev EXPORT_SYMBOL_GPL(inet_ctl_sock_create);
17053d58b5faSDenis V. Lunev
snmp_fold_field(void __percpu * mib,int offt)1706698365faSWANG Cong unsigned long snmp_fold_field(void __percpu *mib, int offt)
17075e0f0435SHerbert Xu {
17085e0f0435SHerbert Xu unsigned long res = 0;
1709698365faSWANG Cong int i;
17105e0f0435SHerbert Xu
1711698365faSWANG Cong for_each_possible_cpu(i)
1712c4c6bc31SRaghavendra K T res += snmp_get_cpu_field(mib, i, offt);
17135e0f0435SHerbert Xu return res;
17145e0f0435SHerbert Xu }
17155e0f0435SHerbert Xu EXPORT_SYMBOL_GPL(snmp_fold_field);
17165e0f0435SHerbert Xu
17174ce3c183SEric Dumazet #if BITS_PER_LONG==32
17184ce3c183SEric Dumazet
snmp_get_cpu_field64(void __percpu * mib,int cpu,int offt,size_t syncp_offset)171980ec1927SDavid S. Miller u64 snmp_get_cpu_field64(void __percpu *mib, int cpu, int offt,
1720c4c6bc31SRaghavendra K T size_t syncp_offset)
17214ce3c183SEric Dumazet {
17228f0ea0feSEric Dumazet void *bhptr;
17234ce3c183SEric Dumazet struct u64_stats_sync *syncp;
17248f0ea0feSEric Dumazet u64 v;
17254ce3c183SEric Dumazet unsigned int start;
17264ce3c183SEric Dumazet
1727698365faSWANG Cong bhptr = per_cpu_ptr(mib, cpu);
17284ce3c183SEric Dumazet syncp = (struct u64_stats_sync *)(bhptr + syncp_offset);
17294ce3c183SEric Dumazet do {
1730d120d1a6SThomas Gleixner start = u64_stats_fetch_begin(syncp);
17318f0ea0feSEric Dumazet v = *(((u64 *)bhptr) + offt);
1732d120d1a6SThomas Gleixner } while (u64_stats_fetch_retry(syncp, start));
17334ce3c183SEric Dumazet
1734c4c6bc31SRaghavendra K T return v;
1735c4c6bc31SRaghavendra K T }
1736c4c6bc31SRaghavendra K T EXPORT_SYMBOL_GPL(snmp_get_cpu_field64);
1737c4c6bc31SRaghavendra K T
snmp_fold_field64(void __percpu * mib,int offt,size_t syncp_offset)1738c4c6bc31SRaghavendra K T u64 snmp_fold_field64(void __percpu *mib, int offt, size_t syncp_offset)
1739c4c6bc31SRaghavendra K T {
1740c4c6bc31SRaghavendra K T u64 res = 0;
1741c4c6bc31SRaghavendra K T int cpu;
1742c4c6bc31SRaghavendra K T
1743c4c6bc31SRaghavendra K T for_each_possible_cpu(cpu) {
1744d1bfc625SMadalin Bucur res += snmp_get_cpu_field64(mib, cpu, offt, syncp_offset);
17454ce3c183SEric Dumazet }
17464ce3c183SEric Dumazet return res;
17474ce3c183SEric Dumazet }
17484ce3c183SEric Dumazet EXPORT_SYMBOL_GPL(snmp_fold_field64);
17494ce3c183SEric Dumazet #endif
17504ce3c183SEric Dumazet
17511da177e4SLinus Torvalds #ifdef CONFIG_IP_MULTICAST
175232613090SAlexey Dobriyan static const struct net_protocol igmp_protocol = {
17531da177e4SLinus Torvalds .handler = igmp_rcv,
17541da177e4SLinus Torvalds };
17551da177e4SLinus Torvalds #endif
17561da177e4SLinus Torvalds
175711052589SKuniyuki Iwashima static const struct net_protocol tcp_protocol = {
17581da177e4SLinus Torvalds .handler = tcp_v4_rcv,
17591da177e4SLinus Torvalds .err_handler = tcp_v4_err,
17601da177e4SLinus Torvalds .no_policy = 1,
17618ed1dc44SHannes Frederic Sowa .icmp_strict_tag_validation = 1,
17621da177e4SLinus Torvalds };
17631da177e4SLinus Torvalds
176411052589SKuniyuki Iwashima static const struct net_protocol udp_protocol = {
17651da177e4SLinus Torvalds .handler = udp_rcv,
17661da177e4SLinus Torvalds .err_handler = udp_err,
17671da177e4SLinus Torvalds .no_policy = 1,
17681da177e4SLinus Torvalds };
17691da177e4SLinus Torvalds
177032613090SAlexey Dobriyan static const struct net_protocol icmp_protocol = {
17711da177e4SLinus Torvalds .handler = icmp_rcv,
17725b052042SLi Wei .err_handler = icmp_err,
17738b7817f3SHerbert Xu .no_policy = 1,
17741da177e4SLinus Torvalds };
17751da177e4SLinus Torvalds
ipv4_mib_init_net(struct net * net)17769b4661bdSPavel Emelyanov static __net_init int ipv4_mib_init_net(struct net *net)
17779b4661bdSPavel Emelyanov {
1778827da44cSJohn Stultz int i;
1779827da44cSJohn Stultz
1780698365faSWANG Cong net->mib.tcp_statistics = alloc_percpu(struct tcp_mib);
1781698365faSWANG Cong if (!net->mib.tcp_statistics)
178257ef42d5SPavel Emelyanov goto err_tcp_mib;
1783698365faSWANG Cong net->mib.ip_statistics = alloc_percpu(struct ipstats_mib);
1784698365faSWANG Cong if (!net->mib.ip_statistics)
1785a20f5799SPavel Emelyanov goto err_ip_mib;
1786827da44cSJohn Stultz
1787827da44cSJohn Stultz for_each_possible_cpu(i) {
1788827da44cSJohn Stultz struct ipstats_mib *af_inet_stats;
1789698365faSWANG Cong af_inet_stats = per_cpu_ptr(net->mib.ip_statistics, i);
1790827da44cSJohn Stultz u64_stats_init(&af_inet_stats->syncp);
1791827da44cSJohn Stultz }
1792827da44cSJohn Stultz
1793698365faSWANG Cong net->mib.net_statistics = alloc_percpu(struct linux_mib);
1794698365faSWANG Cong if (!net->mib.net_statistics)
179561a7e260SPavel Emelyanov goto err_net_mib;
1796698365faSWANG Cong net->mib.udp_statistics = alloc_percpu(struct udp_mib);
1797698365faSWANG Cong if (!net->mib.udp_statistics)
17982f275f91SPavel Emelyanov goto err_udp_mib;
1799698365faSWANG Cong net->mib.udplite_statistics = alloc_percpu(struct udp_mib);
1800698365faSWANG Cong if (!net->mib.udplite_statistics)
1801386019d3SPavel Emelyanov goto err_udplite_mib;
1802698365faSWANG Cong net->mib.icmp_statistics = alloc_percpu(struct icmp_mib);
1803698365faSWANG Cong if (!net->mib.icmp_statistics)
1804b60538a0SPavel Emelyanov goto err_icmp_mib;
1805acb32ba3SEric Dumazet net->mib.icmpmsg_statistics = kzalloc(sizeof(struct icmpmsg_mib),
1806acb32ba3SEric Dumazet GFP_KERNEL);
1807acb32ba3SEric Dumazet if (!net->mib.icmpmsg_statistics)
1808923c6586SPavel Emelyanov goto err_icmpmsg_mib;
180957ef42d5SPavel Emelyanov
181057ef42d5SPavel Emelyanov tcp_mib_init(net);
18119b4661bdSPavel Emelyanov return 0;
181257ef42d5SPavel Emelyanov
1813923c6586SPavel Emelyanov err_icmpmsg_mib:
1814698365faSWANG Cong free_percpu(net->mib.icmp_statistics);
1815b60538a0SPavel Emelyanov err_icmp_mib:
1816698365faSWANG Cong free_percpu(net->mib.udplite_statistics);
1817386019d3SPavel Emelyanov err_udplite_mib:
1818698365faSWANG Cong free_percpu(net->mib.udp_statistics);
18192f275f91SPavel Emelyanov err_udp_mib:
1820698365faSWANG Cong free_percpu(net->mib.net_statistics);
182161a7e260SPavel Emelyanov err_net_mib:
1822698365faSWANG Cong free_percpu(net->mib.ip_statistics);
1823a20f5799SPavel Emelyanov err_ip_mib:
1824698365faSWANG Cong free_percpu(net->mib.tcp_statistics);
182557ef42d5SPavel Emelyanov err_tcp_mib:
182657ef42d5SPavel Emelyanov return -ENOMEM;
18279b4661bdSPavel Emelyanov }
18289b4661bdSPavel Emelyanov
ipv4_mib_exit_net(struct net * net)18299b4661bdSPavel Emelyanov static __net_exit void ipv4_mib_exit_net(struct net *net)
18309b4661bdSPavel Emelyanov {
1831acb32ba3SEric Dumazet kfree(net->mib.icmpmsg_statistics);
1832698365faSWANG Cong free_percpu(net->mib.icmp_statistics);
1833698365faSWANG Cong free_percpu(net->mib.udplite_statistics);
1834698365faSWANG Cong free_percpu(net->mib.udp_statistics);
1835698365faSWANG Cong free_percpu(net->mib.net_statistics);
1836698365faSWANG Cong free_percpu(net->mib.ip_statistics);
1837698365faSWANG Cong free_percpu(net->mib.tcp_statistics);
1838fc518953SFlorian Westphal #ifdef CONFIG_MPTCP
1839fc518953SFlorian Westphal /* allocated on demand, see mptcp_init_sock() */
1840fc518953SFlorian Westphal free_percpu(net->mib.mptcp_statistics);
1841fc518953SFlorian Westphal #endif
18429b4661bdSPavel Emelyanov }
18439b4661bdSPavel Emelyanov
18449b4661bdSPavel Emelyanov static __net_initdata struct pernet_operations ipv4_mib_ops = {
18459b4661bdSPavel Emelyanov .init = ipv4_mib_init_net,
18469b4661bdSPavel Emelyanov .exit = ipv4_mib_exit_net,
18479b4661bdSPavel Emelyanov };
18489b4661bdSPavel Emelyanov
init_ipv4_mibs(void)18491da177e4SLinus Torvalds static int __init init_ipv4_mibs(void)
18501da177e4SLinus Torvalds {
1851d89cbbb1SPavel Emelyanov return register_pernet_subsys(&ipv4_mib_ops);
18521da177e4SLinus Torvalds }
18531da177e4SLinus Torvalds
inet_init_net(struct net * net)1854c9d8f1a6SCong Wang static __net_init int inet_init_net(struct net *net)
1855c9d8f1a6SCong Wang {
1856c9d8f1a6SCong Wang /*
1857c9d8f1a6SCong Wang * Set defaults for local port range
1858c9d8f1a6SCong Wang */
1859c9d8f1a6SCong Wang seqlock_init(&net->ipv4.ip_local_ports.lock);
1860c9d8f1a6SCong Wang net->ipv4.ip_local_ports.range[0] = 32768;
186107f4c900SEric Dumazet net->ipv4.ip_local_ports.range[1] = 60999;
1862ba6b918aSCong Wang
1863ba6b918aSCong Wang seqlock_init(&net->ipv4.ping_group_range.lock);
1864ba6b918aSCong Wang /*
1865ba6b918aSCong Wang * Sane defaults - nobody may create ping sockets.
1866ba6b918aSCong Wang * Boot scripts should set this to distro-specific group.
1867ba6b918aSCong Wang */
1868ba6b918aSCong Wang net->ipv4.ping_group_range.range[0] = make_kgid(&init_user_ns, 1);
1869ba6b918aSCong Wang net->ipv4.ping_group_range.range[1] = make_kgid(&init_user_ns, 0);
1870049bbf58SEzequiel Garcia
1871049bbf58SEzequiel Garcia /* Default values for sysctl-controlled parameters.
1872049bbf58SEzequiel Garcia * We set them here, in case sysctl is not compiled.
1873049bbf58SEzequiel Garcia */
1874049bbf58SEzequiel Garcia net->ipv4.sysctl_ip_default_ttl = IPDEFTTL;
1875432e05d3SPetr Machata net->ipv4.sysctl_ip_fwd_update_priority = 1;
1876049bbf58SEzequiel Garcia net->ipv4.sysctl_ip_dynaddr = 0;
1877049bbf58SEzequiel Garcia net->ipv4.sysctl_ip_early_demux = 1;
1878dddb64bcSsubashab@codeaurora.org net->ipv4.sysctl_udp_early_demux = 1;
1879dddb64bcSsubashab@codeaurora.org net->ipv4.sysctl_tcp_early_demux = 1;
18804f80116dSRoopa Prabhu net->ipv4.sysctl_nexthop_compat_mode = 1;
18814548b683SKrister Johansen #ifdef CONFIG_SYSCTL
18824548b683SKrister Johansen net->ipv4.sysctl_ip_prot_sock = PROT_SOCK;
18834548b683SKrister Johansen #endif
1884049bbf58SEzequiel Garcia
18851714020eSNikolay Borisov /* Some igmp sysctl, whose values are always used */
18861714020eSNikolay Borisov net->ipv4.sysctl_igmp_max_memberships = 20;
18871714020eSNikolay Borisov net->ipv4.sysctl_igmp_max_msf = 10;
18881714020eSNikolay Borisov /* IGMP reports for link-local multicast groups are enabled by default */
18891714020eSNikolay Borisov net->ipv4.sysctl_igmp_llm_reports = 1;
18901714020eSNikolay Borisov net->ipv4.sysctl_igmp_qrv = 2;
18911714020eSNikolay Borisov
1892680aea08SAmit Cohen net->ipv4.sysctl_fib_notify_on_flag_change = 0;
1893680aea08SAmit Cohen
1894c9d8f1a6SCong Wang return 0;
1895c9d8f1a6SCong Wang }
1896c9d8f1a6SCong Wang
1897c9d8f1a6SCong Wang static __net_initdata struct pernet_operations af_inet_ops = {
1898c9d8f1a6SCong Wang .init = inet_init_net,
1899c9d8f1a6SCong Wang };
1900c9d8f1a6SCong Wang
init_inet_pernet_ops(void)1901c9d8f1a6SCong Wang static int __init init_inet_pernet_ops(void)
1902c9d8f1a6SCong Wang {
1903c9d8f1a6SCong Wang return register_pernet_subsys(&af_inet_ops);
1904c9d8f1a6SCong Wang }
1905c9d8f1a6SCong Wang
19061da177e4SLinus Torvalds static int ipv4_proc_init(void);
19071da177e4SLinus Torvalds
190830e224d7SHerbert Xu /*
190930e224d7SHerbert Xu * IP protocol layer initialiser
191030e224d7SHerbert Xu */
191130e224d7SHerbert Xu
191222061d80SVlad Yasevich static struct packet_offload ip_packet_offload __read_mostly = {
191322061d80SVlad Yasevich .type = cpu_to_be16(ETH_P_IP),
1914f191a1d1SVlad Yasevich .callbacks = {
1915f4c50d99SHerbert Xu .gso_segment = inet_gso_segment,
191673cc19f1SHerbert Xu .gro_receive = inet_gro_receive,
191773cc19f1SHerbert Xu .gro_complete = inet_gro_complete,
1918f191a1d1SVlad Yasevich },
191930e224d7SHerbert Xu };
192030e224d7SHerbert Xu
1921cb32f511SEric Dumazet static const struct net_offload ipip_offload = {
1922cb32f511SEric Dumazet .callbacks = {
1923418e897eSWillem de Bruijn .gso_segment = ipip_gso_segment,
1924fac8e0f5SJesse Gross .gro_receive = ipip_gro_receive,
1925b8cba75bSJesse Gross .gro_complete = ipip_gro_complete,
1926cb32f511SEric Dumazet },
1927cb32f511SEric Dumazet };
1928cb32f511SEric Dumazet
ipip_offload_init(void)192993b1b31fSTonghao Zhang static int __init ipip_offload_init(void)
193093b1b31fSTonghao Zhang {
193193b1b31fSTonghao Zhang return inet_add_offload(&ipip_offload, IPPROTO_IPIP);
193293b1b31fSTonghao Zhang }
193393b1b31fSTonghao Zhang
ipv4_offload_init(void)1934808a8f88SVlad Yasevich static int __init ipv4_offload_init(void)
1935808a8f88SVlad Yasevich {
1936808a8f88SVlad Yasevich /*
1937808a8f88SVlad Yasevich * Add offloads
1938808a8f88SVlad Yasevich */
1939da5bab07SDaniel Borkmann if (udpv4_offload_init() < 0)
1940808a8f88SVlad Yasevich pr_crit("%s: Cannot add UDP protocol offload\n", __func__);
194128850dc7SDaniel Borkmann if (tcpv4_offload_init() < 0)
194228850dc7SDaniel Borkmann pr_crit("%s: Cannot add TCP protocol offload\n", __func__);
194393b1b31fSTonghao Zhang if (ipip_offload_init() < 0)
194493b1b31fSTonghao Zhang pr_crit("%s: Cannot add IPIP protocol offload\n", __func__);
1945808a8f88SVlad Yasevich
1946808a8f88SVlad Yasevich dev_add_offload(&ip_packet_offload);
1947808a8f88SVlad Yasevich return 0;
1948808a8f88SVlad Yasevich }
1949808a8f88SVlad Yasevich
1950808a8f88SVlad Yasevich fs_initcall(ipv4_offload_init);
1951808a8f88SVlad Yasevich
1952808a8f88SVlad Yasevich static struct packet_type ip_packet_type __read_mostly = {
1953808a8f88SVlad Yasevich .type = cpu_to_be16(ETH_P_IP),
1954808a8f88SVlad Yasevich .func = ip_rcv,
195517266ee9SEdward Cree .list_func = ip_list_rcv,
1956808a8f88SVlad Yasevich };
1957808a8f88SVlad Yasevich
inet_init(void)19581da177e4SLinus Torvalds static int __init inet_init(void)
19591da177e4SLinus Torvalds {
19601da177e4SLinus Torvalds struct inet_protosw *q;
19611da177e4SLinus Torvalds struct list_head *r;
1962b6246f4dSColin Ian King int rc;
19631da177e4SLinus Torvalds
1964b4772ef8SEyal Birger sock_skb_cb_check_size(sizeof(struct inet_skb_parm));
19651da177e4SLinus Torvalds
19660daf07e5SEric Dumazet raw_hashinfo_init(&raw_v4_hashinfo);
19670daf07e5SEric Dumazet
19681da177e4SLinus Torvalds rc = proto_register(&tcp_prot, 1);
19691da177e4SLinus Torvalds if (rc)
1970122ff243SWANG Cong goto out;
19711da177e4SLinus Torvalds
19721da177e4SLinus Torvalds rc = proto_register(&udp_prot, 1);
19731da177e4SLinus Torvalds if (rc)
19741da177e4SLinus Torvalds goto out_unregister_tcp_proto;
19751da177e4SLinus Torvalds
19761da177e4SLinus Torvalds rc = proto_register(&raw_prot, 1);
19771da177e4SLinus Torvalds if (rc)
19781da177e4SLinus Torvalds goto out_unregister_udp_proto;
19791da177e4SLinus Torvalds
1980c319b4d7SVasiliy Kulikov rc = proto_register(&ping_prot, 1);
1981c319b4d7SVasiliy Kulikov if (rc)
1982c319b4d7SVasiliy Kulikov goto out_unregister_raw_proto;
1983c319b4d7SVasiliy Kulikov
19841da177e4SLinus Torvalds /*
19851da177e4SLinus Torvalds * Tell SOCKET that we are alive...
19861da177e4SLinus Torvalds */
19871da177e4SLinus Torvalds
19881da177e4SLinus Torvalds (void)sock_register(&inet_family_ops);
19891da177e4SLinus Torvalds
1990bd7b1533SAl Viro #ifdef CONFIG_SYSCTL
1991bd7b1533SAl Viro ip_static_sysctl_init();
1992bd7b1533SAl Viro #endif
1993bd7b1533SAl Viro
19941da177e4SLinus Torvalds /*
19951da177e4SLinus Torvalds * Add all the base protocols.
19961da177e4SLinus Torvalds */
19971da177e4SLinus Torvalds
19981da177e4SLinus Torvalds if (inet_add_protocol(&icmp_protocol, IPPROTO_ICMP) < 0)
1999058bd4d2SJoe Perches pr_crit("%s: Cannot add ICMP protocol\n", __func__);
20001da177e4SLinus Torvalds if (inet_add_protocol(&udp_protocol, IPPROTO_UDP) < 0)
2001058bd4d2SJoe Perches pr_crit("%s: Cannot add UDP protocol\n", __func__);
20021da177e4SLinus Torvalds if (inet_add_protocol(&tcp_protocol, IPPROTO_TCP) < 0)
2003058bd4d2SJoe Perches pr_crit("%s: Cannot add TCP protocol\n", __func__);
20041da177e4SLinus Torvalds #ifdef CONFIG_IP_MULTICAST
20051da177e4SLinus Torvalds if (inet_add_protocol(&igmp_protocol, IPPROTO_IGMP) < 0)
2006058bd4d2SJoe Perches pr_crit("%s: Cannot add IGMP protocol\n", __func__);
20071da177e4SLinus Torvalds #endif
20081da177e4SLinus Torvalds
20091da177e4SLinus Torvalds /* Register the socket-side information for inet_create. */
20101da177e4SLinus Torvalds for (r = &inetsw[0]; r < &inetsw[SOCK_MAX]; ++r)
20111da177e4SLinus Torvalds INIT_LIST_HEAD(r);
20121da177e4SLinus Torvalds
20131da177e4SLinus Torvalds for (q = inetsw_array; q < &inetsw_array[INETSW_ARRAY_LEN]; ++q)
20141da177e4SLinus Torvalds inet_register_protosw(q);
20151da177e4SLinus Torvalds
20161da177e4SLinus Torvalds /*
20171da177e4SLinus Torvalds * Set the ARP module up
20181da177e4SLinus Torvalds */
20191da177e4SLinus Torvalds
20201da177e4SLinus Torvalds arp_init();
20211da177e4SLinus Torvalds
20221da177e4SLinus Torvalds /*
20231da177e4SLinus Torvalds * Set the IP module up
20241da177e4SLinus Torvalds */
20251da177e4SLinus Torvalds
20261da177e4SLinus Torvalds ip_init();
20271da177e4SLinus Torvalds
2028e22e45fcSMuchun Song /* Initialise per-cpu ipv4 mibs */
2029e22e45fcSMuchun Song if (init_ipv4_mibs())
2030e22e45fcSMuchun Song panic("%s: Cannot init ipv4 mibs\n", __func__);
2031e22e45fcSMuchun Song
20321da177e4SLinus Torvalds /* Setup TCP slab cache for open requests. */
20331da177e4SLinus Torvalds tcp_init();
20341da177e4SLinus Torvalds
203595766fffSHideo Aoki /* Setup UDP memory threshold */
203695766fffSHideo Aoki udp_init();
203795766fffSHideo Aoki
2038ba4e58ecSGerrit Renker /* Add UDP-Lite (RFC 3828) */
2039ba4e58ecSGerrit Renker udplite4_register();
20401da177e4SLinus Torvalds
20416897445fSMike Manning raw_init();
20426897445fSMike Manning
2043c319b4d7SVasiliy Kulikov ping_init();
2044c319b4d7SVasiliy Kulikov
20451da177e4SLinus Torvalds /*
20461da177e4SLinus Torvalds * Set the ICMP layer up
20471da177e4SLinus Torvalds */
20481da177e4SLinus Torvalds
2049a5710d65SDenis V. Lunev if (icmp_init() < 0)
2050a5710d65SDenis V. Lunev panic("Failed to create the ICMP control socket.\n");
20511da177e4SLinus Torvalds
20521da177e4SLinus Torvalds /*
20531da177e4SLinus Torvalds * Initialise the multicast router
20541da177e4SLinus Torvalds */
20551da177e4SLinus Torvalds #if defined(CONFIG_IP_MROUTE)
205603d2f897SWang Chen if (ip_mr_init())
2057058bd4d2SJoe Perches pr_crit("%s: Cannot init ipv4 mroute\n", __func__);
20581da177e4SLinus Torvalds #endif
2059c9d8f1a6SCong Wang
2060c9d8f1a6SCong Wang if (init_inet_pernet_ops())
2061c9d8f1a6SCong Wang pr_crit("%s: Cannot init ipv4 inet pernet ops\n", __func__);
20621da177e4SLinus Torvalds
20631da177e4SLinus Torvalds ipv4_proc_init();
20641da177e4SLinus Torvalds
20651da177e4SLinus Torvalds ipfrag_init();
20661da177e4SLinus Torvalds
206730e224d7SHerbert Xu dev_add_pack(&ip_packet_type);
206830e224d7SHerbert Xu
2069045a0fa0SThomas Graf ip_tunnel_core_init();
2070045a0fa0SThomas Graf
20711da177e4SLinus Torvalds rc = 0;
20721da177e4SLinus Torvalds out:
20731da177e4SLinus Torvalds return rc;
2074c319b4d7SVasiliy Kulikov out_unregister_raw_proto:
2075c319b4d7SVasiliy Kulikov proto_unregister(&raw_prot);
20761da177e4SLinus Torvalds out_unregister_udp_proto:
20771da177e4SLinus Torvalds proto_unregister(&udp_prot);
2078321efff7SOlaf Kirch out_unregister_tcp_proto:
2079321efff7SOlaf Kirch proto_unregister(&tcp_prot);
20801da177e4SLinus Torvalds goto out;
20811da177e4SLinus Torvalds }
20821da177e4SLinus Torvalds
2083a536e077SHeiko Carstens fs_initcall(inet_init);
20841da177e4SLinus Torvalds
20851da177e4SLinus Torvalds /* ------------------------------------------------------------------------ */
20861da177e4SLinus Torvalds
20871da177e4SLinus Torvalds #ifdef CONFIG_PROC_FS
ipv4_proc_init(void)20881da177e4SLinus Torvalds static int __init ipv4_proc_init(void)
20891da177e4SLinus Torvalds {
20901da177e4SLinus Torvalds int rc = 0;
20911da177e4SLinus Torvalds
20921da177e4SLinus Torvalds if (raw_proc_init())
20931da177e4SLinus Torvalds goto out_raw;
20941da177e4SLinus Torvalds if (tcp4_proc_init())
20951da177e4SLinus Torvalds goto out_tcp;
20961da177e4SLinus Torvalds if (udp4_proc_init())
20971da177e4SLinus Torvalds goto out_udp;
2098c319b4d7SVasiliy Kulikov if (ping_proc_init())
2099c319b4d7SVasiliy Kulikov goto out_ping;
21001da177e4SLinus Torvalds if (ip_misc_proc_init())
21011da177e4SLinus Torvalds goto out_misc;
21021da177e4SLinus Torvalds out:
21031da177e4SLinus Torvalds return rc;
21041da177e4SLinus Torvalds out_misc:
2105c319b4d7SVasiliy Kulikov ping_proc_exit();
2106c319b4d7SVasiliy Kulikov out_ping:
21071da177e4SLinus Torvalds udp4_proc_exit();
21081da177e4SLinus Torvalds out_udp:
21091da177e4SLinus Torvalds tcp4_proc_exit();
21101da177e4SLinus Torvalds out_tcp:
21111da177e4SLinus Torvalds raw_proc_exit();
21121da177e4SLinus Torvalds out_raw:
21131da177e4SLinus Torvalds rc = -ENOMEM;
21141da177e4SLinus Torvalds goto out;
21151da177e4SLinus Torvalds }
21161da177e4SLinus Torvalds
21171da177e4SLinus Torvalds #else /* CONFIG_PROC_FS */
ipv4_proc_init(void)21181da177e4SLinus Torvalds static int __init ipv4_proc_init(void)
21191da177e4SLinus Torvalds {
21201da177e4SLinus Torvalds return 0;
21211da177e4SLinus Torvalds }
21221da177e4SLinus Torvalds #endif /* CONFIG_PROC_FS */
2123