xref: /openbmc/linux/net/mptcp/protocol.c (revision 14c441b564d560dea4c93947d5b40a992e13ca31)
1f870fa0bSMat Martineau // SPDX-License-Identifier: GPL-2.0
2f870fa0bSMat Martineau /* Multipath TCP
3f870fa0bSMat Martineau  *
4f870fa0bSMat Martineau  * Copyright (c) 2017 - 2019, Intel Corporation.
5f870fa0bSMat Martineau  */
6f870fa0bSMat Martineau 
7f870fa0bSMat Martineau #define pr_fmt(fmt) "MPTCP: " fmt
8f870fa0bSMat Martineau 
9f870fa0bSMat Martineau #include <linux/kernel.h>
10f870fa0bSMat Martineau #include <linux/module.h>
11f870fa0bSMat Martineau #include <linux/netdevice.h>
127a6a6cbcSPaolo Abeni #include <linux/sched/signal.h>
137a6a6cbcSPaolo Abeni #include <linux/atomic.h>
14f870fa0bSMat Martineau #include <net/sock.h>
15f870fa0bSMat Martineau #include <net/inet_common.h>
16f870fa0bSMat Martineau #include <net/inet_hashtables.h>
17f870fa0bSMat Martineau #include <net/protocol.h>
18f870fa0bSMat Martineau #include <net/tcp.h>
19cf7da0d6SPeter Krystad #if IS_ENABLED(CONFIG_MPTCP_IPV6)
20cf7da0d6SPeter Krystad #include <net/transp_v6.h>
21cf7da0d6SPeter Krystad #endif
22f870fa0bSMat Martineau #include <net/mptcp.h>
23f870fa0bSMat Martineau #include "protocol.h"
24f870fa0bSMat Martineau 
252303f994SPeter Krystad #define MPTCP_SAME_STATE TCP_MAX_STATES
262303f994SPeter Krystad 
27b0519de8SFlorian Westphal #if IS_ENABLED(CONFIG_MPTCP_IPV6)
28b0519de8SFlorian Westphal struct mptcp6_sock {
29b0519de8SFlorian Westphal 	struct mptcp_sock msk;
30b0519de8SFlorian Westphal 	struct ipv6_pinfo np;
31b0519de8SFlorian Westphal };
32b0519de8SFlorian Westphal #endif
33b0519de8SFlorian Westphal 
346771bfd9SFlorian Westphal struct mptcp_skb_cb {
356771bfd9SFlorian Westphal 	u32 offset;
366771bfd9SFlorian Westphal };
376771bfd9SFlorian Westphal 
386771bfd9SFlorian Westphal #define MPTCP_SKB_CB(__skb)	((struct mptcp_skb_cb *)&((__skb)->cb[0]))
396771bfd9SFlorian Westphal 
402303f994SPeter Krystad /* If msk has an initial subflow socket, and the MP_CAPABLE handshake has not
412303f994SPeter Krystad  * completed yet or has failed, return the subflow socket.
422303f994SPeter Krystad  * Otherwise return NULL.
432303f994SPeter Krystad  */
442303f994SPeter Krystad static struct socket *__mptcp_nmpc_socket(const struct mptcp_sock *msk)
452303f994SPeter Krystad {
46d22f4988SChristoph Paasch 	if (!msk->subflow || READ_ONCE(msk->can_ack))
472303f994SPeter Krystad 		return NULL;
482303f994SPeter Krystad 
492303f994SPeter Krystad 	return msk->subflow;
502303f994SPeter Krystad }
512303f994SPeter Krystad 
528ab183deSPaolo Abeni static bool __mptcp_needs_tcp_fallback(const struct mptcp_sock *msk)
538ab183deSPaolo Abeni {
548ab183deSPaolo Abeni 	return msk->first && !sk_is_mptcp(msk->first);
558ab183deSPaolo Abeni }
568ab183deSPaolo Abeni 
578ab183deSPaolo Abeni static struct socket *__mptcp_tcp_fallback(struct mptcp_sock *msk)
58cec37a6eSPeter Krystad {
59cec37a6eSPeter Krystad 	sock_owned_by_me((const struct sock *)msk);
60cec37a6eSPeter Krystad 
618ab183deSPaolo Abeni 	if (likely(!__mptcp_needs_tcp_fallback(msk)))
62cec37a6eSPeter Krystad 		return NULL;
63cec37a6eSPeter Krystad 
648ab183deSPaolo Abeni 	if (msk->subflow) {
652c22c06cSFlorian Westphal 		release_sock((struct sock *)msk);
662c22c06cSFlorian Westphal 		return msk->subflow;
678ab183deSPaolo Abeni 	}
688ab183deSPaolo Abeni 
692c22c06cSFlorian Westphal 	return NULL;
70cec37a6eSPeter Krystad }
71cec37a6eSPeter Krystad 
722303f994SPeter Krystad static bool __mptcp_can_create_subflow(const struct mptcp_sock *msk)
732303f994SPeter Krystad {
748ab183deSPaolo Abeni 	return !msk->first;
752303f994SPeter Krystad }
762303f994SPeter Krystad 
772303f994SPeter Krystad static struct socket *__mptcp_socket_create(struct mptcp_sock *msk, int state)
782303f994SPeter Krystad {
792303f994SPeter Krystad 	struct mptcp_subflow_context *subflow;
802303f994SPeter Krystad 	struct sock *sk = (struct sock *)msk;
812303f994SPeter Krystad 	struct socket *ssock;
822303f994SPeter Krystad 	int err;
832303f994SPeter Krystad 
842303f994SPeter Krystad 	ssock = __mptcp_nmpc_socket(msk);
852303f994SPeter Krystad 	if (ssock)
862303f994SPeter Krystad 		goto set_state;
872303f994SPeter Krystad 
882303f994SPeter Krystad 	if (!__mptcp_can_create_subflow(msk))
892303f994SPeter Krystad 		return ERR_PTR(-EINVAL);
902303f994SPeter Krystad 
912303f994SPeter Krystad 	err = mptcp_subflow_create_socket(sk, &ssock);
922303f994SPeter Krystad 	if (err)
932303f994SPeter Krystad 		return ERR_PTR(err);
942303f994SPeter Krystad 
958ab183deSPaolo Abeni 	msk->first = ssock->sk;
962303f994SPeter Krystad 	msk->subflow = ssock;
972303f994SPeter Krystad 	subflow = mptcp_subflow_ctx(ssock->sk);
98cec37a6eSPeter Krystad 	list_add(&subflow->node, &msk->conn_list);
992303f994SPeter Krystad 	subflow->request_mptcp = 1;
1002303f994SPeter Krystad 
1012303f994SPeter Krystad set_state:
1022303f994SPeter Krystad 	if (state != MPTCP_SAME_STATE)
1032303f994SPeter Krystad 		inet_sk_state_store(sk, state);
1042303f994SPeter Krystad 	return ssock;
1052303f994SPeter Krystad }
1062303f994SPeter Krystad 
107cec37a6eSPeter Krystad static struct sock *mptcp_subflow_get(const struct mptcp_sock *msk)
108cec37a6eSPeter Krystad {
109cec37a6eSPeter Krystad 	struct mptcp_subflow_context *subflow;
110cec37a6eSPeter Krystad 
111cec37a6eSPeter Krystad 	sock_owned_by_me((const struct sock *)msk);
112cec37a6eSPeter Krystad 
113cec37a6eSPeter Krystad 	mptcp_for_each_subflow(msk, subflow) {
114cec37a6eSPeter Krystad 		return mptcp_subflow_tcp_sock(subflow);
115cec37a6eSPeter Krystad 	}
116cec37a6eSPeter Krystad 
117cec37a6eSPeter Krystad 	return NULL;
118cec37a6eSPeter Krystad }
119cec37a6eSPeter Krystad 
1206771bfd9SFlorian Westphal static void __mptcp_move_skb(struct mptcp_sock *msk, struct sock *ssk,
1216771bfd9SFlorian Westphal 			     struct sk_buff *skb,
1226771bfd9SFlorian Westphal 			     unsigned int offset, size_t copy_len)
1236771bfd9SFlorian Westphal {
1246771bfd9SFlorian Westphal 	struct sock *sk = (struct sock *)msk;
1256771bfd9SFlorian Westphal 
1266771bfd9SFlorian Westphal 	__skb_unlink(skb, &ssk->sk_receive_queue);
127600911ffSFlorian Westphal 	skb_set_owner_r(skb, sk);
1286771bfd9SFlorian Westphal 	__skb_queue_tail(&sk->sk_receive_queue, skb);
1296771bfd9SFlorian Westphal 
1306771bfd9SFlorian Westphal 	msk->ack_seq += copy_len;
1316771bfd9SFlorian Westphal 	MPTCP_SKB_CB(skb)->offset = offset;
1326771bfd9SFlorian Westphal }
1336771bfd9SFlorian Westphal 
1346771bfd9SFlorian Westphal static bool __mptcp_move_skbs_from_subflow(struct mptcp_sock *msk,
1356771bfd9SFlorian Westphal 					   struct sock *ssk,
1366771bfd9SFlorian Westphal 					   unsigned int *bytes)
1376771bfd9SFlorian Westphal {
1386771bfd9SFlorian Westphal 	struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk);
139600911ffSFlorian Westphal 	struct sock *sk = (struct sock *)msk;
1406771bfd9SFlorian Westphal 	unsigned int moved = 0;
1416771bfd9SFlorian Westphal 	bool more_data_avail;
1426771bfd9SFlorian Westphal 	struct tcp_sock *tp;
1436771bfd9SFlorian Westphal 	bool done = false;
144600911ffSFlorian Westphal 	int rcvbuf;
145600911ffSFlorian Westphal 
146600911ffSFlorian Westphal 	rcvbuf = max(ssk->sk_rcvbuf, sk->sk_rcvbuf);
147600911ffSFlorian Westphal 	if (rcvbuf > sk->sk_rcvbuf)
148600911ffSFlorian Westphal 		sk->sk_rcvbuf = rcvbuf;
1496771bfd9SFlorian Westphal 
1506771bfd9SFlorian Westphal 	tp = tcp_sk(ssk);
1516771bfd9SFlorian Westphal 	do {
1526771bfd9SFlorian Westphal 		u32 map_remaining, offset;
1536771bfd9SFlorian Westphal 		u32 seq = tp->copied_seq;
1546771bfd9SFlorian Westphal 		struct sk_buff *skb;
1556771bfd9SFlorian Westphal 		bool fin;
1566771bfd9SFlorian Westphal 
1576771bfd9SFlorian Westphal 		/* try to move as much data as available */
1586771bfd9SFlorian Westphal 		map_remaining = subflow->map_data_len -
1596771bfd9SFlorian Westphal 				mptcp_subflow_get_map_offset(subflow);
1606771bfd9SFlorian Westphal 
1616771bfd9SFlorian Westphal 		skb = skb_peek(&ssk->sk_receive_queue);
1626771bfd9SFlorian Westphal 		if (!skb)
1636771bfd9SFlorian Westphal 			break;
1646771bfd9SFlorian Westphal 
1656771bfd9SFlorian Westphal 		offset = seq - TCP_SKB_CB(skb)->seq;
1666771bfd9SFlorian Westphal 		fin = TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN;
1676771bfd9SFlorian Westphal 		if (fin) {
1686771bfd9SFlorian Westphal 			done = true;
1696771bfd9SFlorian Westphal 			seq++;
1706771bfd9SFlorian Westphal 		}
1716771bfd9SFlorian Westphal 
1726771bfd9SFlorian Westphal 		if (offset < skb->len) {
1736771bfd9SFlorian Westphal 			size_t len = skb->len - offset;
1746771bfd9SFlorian Westphal 
1756771bfd9SFlorian Westphal 			if (tp->urg_data)
1766771bfd9SFlorian Westphal 				done = true;
1776771bfd9SFlorian Westphal 
1786771bfd9SFlorian Westphal 			__mptcp_move_skb(msk, ssk, skb, offset, len);
1796771bfd9SFlorian Westphal 			seq += len;
1806771bfd9SFlorian Westphal 			moved += len;
1816771bfd9SFlorian Westphal 
1826771bfd9SFlorian Westphal 			if (WARN_ON_ONCE(map_remaining < len))
1836771bfd9SFlorian Westphal 				break;
1846771bfd9SFlorian Westphal 		} else {
1856771bfd9SFlorian Westphal 			WARN_ON_ONCE(!fin);
1866771bfd9SFlorian Westphal 			sk_eat_skb(ssk, skb);
1876771bfd9SFlorian Westphal 			done = true;
1886771bfd9SFlorian Westphal 		}
1896771bfd9SFlorian Westphal 
1906771bfd9SFlorian Westphal 		WRITE_ONCE(tp->copied_seq, seq);
1916771bfd9SFlorian Westphal 		more_data_avail = mptcp_subflow_data_available(ssk);
192600911ffSFlorian Westphal 
193600911ffSFlorian Westphal 		if (atomic_read(&sk->sk_rmem_alloc) > READ_ONCE(sk->sk_rcvbuf)) {
194600911ffSFlorian Westphal 			done = true;
195600911ffSFlorian Westphal 			break;
196600911ffSFlorian Westphal 		}
1976771bfd9SFlorian Westphal 	} while (more_data_avail);
1986771bfd9SFlorian Westphal 
1996771bfd9SFlorian Westphal 	*bytes = moved;
2006771bfd9SFlorian Westphal 
2016771bfd9SFlorian Westphal 	return done;
2026771bfd9SFlorian Westphal }
2036771bfd9SFlorian Westphal 
2042e52213cSFlorian Westphal /* In most cases we will be able to lock the mptcp socket.  If its already
2052e52213cSFlorian Westphal  * owned, we need to defer to the work queue to avoid ABBA deadlock.
2062e52213cSFlorian Westphal  */
2072e52213cSFlorian Westphal static bool move_skbs_to_msk(struct mptcp_sock *msk, struct sock *ssk)
2082e52213cSFlorian Westphal {
2092e52213cSFlorian Westphal 	struct sock *sk = (struct sock *)msk;
2102e52213cSFlorian Westphal 	unsigned int moved = 0;
2112e52213cSFlorian Westphal 
2122e52213cSFlorian Westphal 	if (READ_ONCE(sk->sk_lock.owned))
2132e52213cSFlorian Westphal 		return false;
2142e52213cSFlorian Westphal 
2152e52213cSFlorian Westphal 	if (unlikely(!spin_trylock_bh(&sk->sk_lock.slock)))
2162e52213cSFlorian Westphal 		return false;
2172e52213cSFlorian Westphal 
2182e52213cSFlorian Westphal 	/* must re-check after taking the lock */
2192e52213cSFlorian Westphal 	if (!READ_ONCE(sk->sk_lock.owned))
2202e52213cSFlorian Westphal 		__mptcp_move_skbs_from_subflow(msk, ssk, &moved);
2212e52213cSFlorian Westphal 
2222e52213cSFlorian Westphal 	spin_unlock_bh(&sk->sk_lock.slock);
2232e52213cSFlorian Westphal 
2242e52213cSFlorian Westphal 	return moved > 0;
2252e52213cSFlorian Westphal }
2262e52213cSFlorian Westphal 
2272e52213cSFlorian Westphal void mptcp_data_ready(struct sock *sk, struct sock *ssk)
228101f6f85SFlorian Westphal {
229101f6f85SFlorian Westphal 	struct mptcp_sock *msk = mptcp_sk(sk);
230101f6f85SFlorian Westphal 
231101f6f85SFlorian Westphal 	set_bit(MPTCP_DATA_READY, &msk->flags);
2326771bfd9SFlorian Westphal 
2332e52213cSFlorian Westphal 	if (atomic_read(&sk->sk_rmem_alloc) < READ_ONCE(sk->sk_rcvbuf) &&
2342e52213cSFlorian Westphal 	    move_skbs_to_msk(msk, ssk))
2352e52213cSFlorian Westphal 		goto wake;
2362e52213cSFlorian Westphal 
237600911ffSFlorian Westphal 	/* don't schedule if mptcp sk is (still) over limit */
238600911ffSFlorian Westphal 	if (atomic_read(&sk->sk_rmem_alloc) > READ_ONCE(sk->sk_rcvbuf))
239600911ffSFlorian Westphal 		goto wake;
240600911ffSFlorian Westphal 
241*14c441b5SPaolo Abeni 	/* mptcp socket is owned, release_cb should retry */
242*14c441b5SPaolo Abeni 	if (!test_and_set_bit(TCP_DELACK_TIMER_DEFERRED,
243*14c441b5SPaolo Abeni 			      &sk->sk_tsq_flags)) {
244*14c441b5SPaolo Abeni 		sock_hold(sk);
2456771bfd9SFlorian Westphal 
246*14c441b5SPaolo Abeni 		/* need to try again, its possible release_cb() has already
247*14c441b5SPaolo Abeni 		 * been called after the test_and_set_bit() above.
248*14c441b5SPaolo Abeni 		 */
249*14c441b5SPaolo Abeni 		move_skbs_to_msk(msk, ssk);
250*14c441b5SPaolo Abeni 	}
251600911ffSFlorian Westphal wake:
252101f6f85SFlorian Westphal 	sk->sk_data_ready(sk);
253101f6f85SFlorian Westphal }
254101f6f85SFlorian Westphal 
2556d0060f6SMat Martineau static bool mptcp_ext_cache_refill(struct mptcp_sock *msk)
2566d0060f6SMat Martineau {
2576d0060f6SMat Martineau 	if (!msk->cached_ext)
2586d0060f6SMat Martineau 		msk->cached_ext = __skb_ext_alloc();
2596d0060f6SMat Martineau 
2606d0060f6SMat Martineau 	return !!msk->cached_ext;
2616d0060f6SMat Martineau }
2626d0060f6SMat Martineau 
2637a6a6cbcSPaolo Abeni static struct sock *mptcp_subflow_recv_lookup(const struct mptcp_sock *msk)
2647a6a6cbcSPaolo Abeni {
2657a6a6cbcSPaolo Abeni 	struct mptcp_subflow_context *subflow;
2667a6a6cbcSPaolo Abeni 	struct sock *sk = (struct sock *)msk;
2677a6a6cbcSPaolo Abeni 
2687a6a6cbcSPaolo Abeni 	sock_owned_by_me(sk);
2697a6a6cbcSPaolo Abeni 
2707a6a6cbcSPaolo Abeni 	mptcp_for_each_subflow(msk, subflow) {
2717a6a6cbcSPaolo Abeni 		if (subflow->data_avail)
2727a6a6cbcSPaolo Abeni 			return mptcp_subflow_tcp_sock(subflow);
2737a6a6cbcSPaolo Abeni 	}
2747a6a6cbcSPaolo Abeni 
2757a6a6cbcSPaolo Abeni 	return NULL;
2767a6a6cbcSPaolo Abeni }
2777a6a6cbcSPaolo Abeni 
27857040755SPaolo Abeni static inline bool mptcp_skb_can_collapse_to(const struct mptcp_sock *msk,
27957040755SPaolo Abeni 					     const struct sk_buff *skb,
28057040755SPaolo Abeni 					     const struct mptcp_ext *mpext)
2816d0060f6SMat Martineau {
28257040755SPaolo Abeni 	if (!tcp_skb_can_collapse_to(skb))
28357040755SPaolo Abeni 		return false;
28457040755SPaolo Abeni 
28557040755SPaolo Abeni 	/* can collapse only if MPTCP level sequence is in order */
28657040755SPaolo Abeni 	return mpext && mpext->data_seq + mpext->data_len == msk->write_seq;
28757040755SPaolo Abeni }
28857040755SPaolo Abeni 
28957040755SPaolo Abeni static int mptcp_sendmsg_frag(struct sock *sk, struct sock *ssk,
29057040755SPaolo Abeni 			      struct msghdr *msg, long *timeo, int *pmss_now,
29157040755SPaolo Abeni 			      int *ps_goal)
29257040755SPaolo Abeni {
29357040755SPaolo Abeni 	int mss_now, avail_size, size_goal, ret;
2946d0060f6SMat Martineau 	struct mptcp_sock *msk = mptcp_sk(sk);
2956d0060f6SMat Martineau 	struct mptcp_ext *mpext = NULL;
29657040755SPaolo Abeni 	struct sk_buff *skb, *tail;
29757040755SPaolo Abeni 	bool can_collapse = false;
2986d0060f6SMat Martineau 	struct page_frag *pfrag;
2996d0060f6SMat Martineau 	size_t psize;
3006d0060f6SMat Martineau 
3016d0060f6SMat Martineau 	/* use the mptcp page cache so that we can easily move the data
3026d0060f6SMat Martineau 	 * from one substream to another, but do per subflow memory accounting
3036d0060f6SMat Martineau 	 */
3046d0060f6SMat Martineau 	pfrag = sk_page_frag(sk);
3056d0060f6SMat Martineau 	while (!sk_page_frag_refill(ssk, pfrag) ||
3066d0060f6SMat Martineau 	       !mptcp_ext_cache_refill(msk)) {
3076d0060f6SMat Martineau 		ret = sk_stream_wait_memory(ssk, timeo);
3086d0060f6SMat Martineau 		if (ret)
3096d0060f6SMat Martineau 			return ret;
3108ab183deSPaolo Abeni 		if (unlikely(__mptcp_needs_tcp_fallback(msk)))
3118ab183deSPaolo Abeni 			return 0;
3126d0060f6SMat Martineau 	}
3136d0060f6SMat Martineau 
3146d0060f6SMat Martineau 	/* compute copy limit */
3156d0060f6SMat Martineau 	mss_now = tcp_send_mss(ssk, &size_goal, msg->msg_flags);
31657040755SPaolo Abeni 	*pmss_now = mss_now;
31757040755SPaolo Abeni 	*ps_goal = size_goal;
31857040755SPaolo Abeni 	avail_size = size_goal;
31957040755SPaolo Abeni 	skb = tcp_write_queue_tail(ssk);
32057040755SPaolo Abeni 	if (skb) {
32157040755SPaolo Abeni 		mpext = skb_ext_find(skb, SKB_EXT_MPTCP);
3226d0060f6SMat Martineau 
32357040755SPaolo Abeni 		/* Limit the write to the size available in the
32457040755SPaolo Abeni 		 * current skb, if any, so that we create at most a new skb.
32557040755SPaolo Abeni 		 * Explicitly tells TCP internals to avoid collapsing on later
32657040755SPaolo Abeni 		 * queue management operation, to avoid breaking the ext <->
32757040755SPaolo Abeni 		 * SSN association set here
32857040755SPaolo Abeni 		 */
32957040755SPaolo Abeni 		can_collapse = (size_goal - skb->len > 0) &&
33057040755SPaolo Abeni 			      mptcp_skb_can_collapse_to(msk, skb, mpext);
33157040755SPaolo Abeni 		if (!can_collapse)
33257040755SPaolo Abeni 			TCP_SKB_CB(skb)->eor = 1;
33357040755SPaolo Abeni 		else
33457040755SPaolo Abeni 			avail_size = size_goal - skb->len;
33557040755SPaolo Abeni 	}
33657040755SPaolo Abeni 	psize = min_t(size_t, pfrag->size - pfrag->offset, avail_size);
33757040755SPaolo Abeni 
33857040755SPaolo Abeni 	/* Copy to page */
3396d0060f6SMat Martineau 	pr_debug("left=%zu", msg_data_left(msg));
3406d0060f6SMat Martineau 	psize = copy_page_from_iter(pfrag->page, pfrag->offset,
3416d0060f6SMat Martineau 				    min_t(size_t, msg_data_left(msg), psize),
3426d0060f6SMat Martineau 				    &msg->msg_iter);
3436d0060f6SMat Martineau 	pr_debug("left=%zu", msg_data_left(msg));
3446d0060f6SMat Martineau 	if (!psize)
3456d0060f6SMat Martineau 		return -EINVAL;
3466d0060f6SMat Martineau 
34757040755SPaolo Abeni 	/* tell the TCP stack to delay the push so that we can safely
34857040755SPaolo Abeni 	 * access the skb after the sendpages call
3496d0060f6SMat Martineau 	 */
3506d0060f6SMat Martineau 	ret = do_tcp_sendpages(ssk, pfrag->page, pfrag->offset, psize,
3516d0060f6SMat Martineau 			       msg->msg_flags | MSG_SENDPAGE_NOTLAST);
3526d0060f6SMat Martineau 	if (ret <= 0)
3536d0060f6SMat Martineau 		return ret;
3546d0060f6SMat Martineau 	if (unlikely(ret < psize))
3556d0060f6SMat Martineau 		iov_iter_revert(&msg->msg_iter, psize - ret);
3566d0060f6SMat Martineau 
35757040755SPaolo Abeni 	/* if the tail skb extension is still the cached one, collapsing
35857040755SPaolo Abeni 	 * really happened. Note: we can't check for 'same skb' as the sk_buff
35957040755SPaolo Abeni 	 * hdr on tail can be transmitted, freed and re-allocated by the
36057040755SPaolo Abeni 	 * do_tcp_sendpages() call
36157040755SPaolo Abeni 	 */
36257040755SPaolo Abeni 	tail = tcp_write_queue_tail(ssk);
36357040755SPaolo Abeni 	if (mpext && tail && mpext == skb_ext_find(tail, SKB_EXT_MPTCP)) {
36457040755SPaolo Abeni 		WARN_ON_ONCE(!can_collapse);
36557040755SPaolo Abeni 		mpext->data_len += ret;
36657040755SPaolo Abeni 		goto out;
36757040755SPaolo Abeni 	}
36857040755SPaolo Abeni 
3696d0060f6SMat Martineau 	skb = tcp_write_queue_tail(ssk);
3706d0060f6SMat Martineau 	mpext = __skb_ext_set(skb, SKB_EXT_MPTCP, msk->cached_ext);
3716d0060f6SMat Martineau 	msk->cached_ext = NULL;
3726d0060f6SMat Martineau 
3736d0060f6SMat Martineau 	memset(mpext, 0, sizeof(*mpext));
3746d0060f6SMat Martineau 	mpext->data_seq = msk->write_seq;
3756d0060f6SMat Martineau 	mpext->subflow_seq = mptcp_subflow_ctx(ssk)->rel_write_seq;
3766d0060f6SMat Martineau 	mpext->data_len = ret;
3776d0060f6SMat Martineau 	mpext->use_map = 1;
3786d0060f6SMat Martineau 	mpext->dsn64 = 1;
3796d0060f6SMat Martineau 
3806d0060f6SMat Martineau 	pr_debug("data_seq=%llu subflow_seq=%u data_len=%u dsn64=%d",
3816d0060f6SMat Martineau 		 mpext->data_seq, mpext->subflow_seq, mpext->data_len,
3826d0060f6SMat Martineau 		 mpext->dsn64);
3836d0060f6SMat Martineau 
38457040755SPaolo Abeni out:
3856d0060f6SMat Martineau 	pfrag->offset += ret;
3866d0060f6SMat Martineau 	msk->write_seq += ret;
3876d0060f6SMat Martineau 	mptcp_subflow_ctx(ssk)->rel_write_seq += ret;
3886d0060f6SMat Martineau 
3896d0060f6SMat Martineau 	return ret;
3906d0060f6SMat Martineau }
3916d0060f6SMat Martineau 
3921891c4a0SFlorian Westphal static void ssk_check_wmem(struct mptcp_sock *msk, struct sock *ssk)
3931891c4a0SFlorian Westphal {
3941891c4a0SFlorian Westphal 	struct socket *sock;
3951891c4a0SFlorian Westphal 
3961891c4a0SFlorian Westphal 	if (likely(sk_stream_is_writeable(ssk)))
3971891c4a0SFlorian Westphal 		return;
3981891c4a0SFlorian Westphal 
3991891c4a0SFlorian Westphal 	sock = READ_ONCE(ssk->sk_socket);
4001891c4a0SFlorian Westphal 
4011891c4a0SFlorian Westphal 	if (sock) {
4021891c4a0SFlorian Westphal 		clear_bit(MPTCP_SEND_SPACE, &msk->flags);
4031891c4a0SFlorian Westphal 		smp_mb__after_atomic();
4041891c4a0SFlorian Westphal 		/* set NOSPACE only after clearing SEND_SPACE flag */
4051891c4a0SFlorian Westphal 		set_bit(SOCK_NOSPACE, &sock->flags);
4061891c4a0SFlorian Westphal 	}
4071891c4a0SFlorian Westphal }
4081891c4a0SFlorian Westphal 
409f870fa0bSMat Martineau static int mptcp_sendmsg(struct sock *sk, struct msghdr *msg, size_t len)
410f870fa0bSMat Martineau {
41157040755SPaolo Abeni 	int mss_now = 0, size_goal = 0, ret = 0;
412f870fa0bSMat Martineau 	struct mptcp_sock *msk = mptcp_sk(sk);
413cec37a6eSPeter Krystad 	struct socket *ssock;
4146d0060f6SMat Martineau 	size_t copied = 0;
415cec37a6eSPeter Krystad 	struct sock *ssk;
4166d0060f6SMat Martineau 	long timeo;
417f870fa0bSMat Martineau 
418f870fa0bSMat Martineau 	if (msg->msg_flags & ~(MSG_MORE | MSG_DONTWAIT | MSG_NOSIGNAL))
419f870fa0bSMat Martineau 		return -EOPNOTSUPP;
420f870fa0bSMat Martineau 
421cec37a6eSPeter Krystad 	lock_sock(sk);
422cec37a6eSPeter Krystad 	ssock = __mptcp_tcp_fallback(msk);
4238ab183deSPaolo Abeni 	if (unlikely(ssock)) {
4248ab183deSPaolo Abeni fallback:
425cec37a6eSPeter Krystad 		pr_debug("fallback passthrough");
426cec37a6eSPeter Krystad 		ret = sock_sendmsg(ssock, msg);
4278ab183deSPaolo Abeni 		return ret >= 0 ? ret + copied : (copied ? copied : ret);
428cec37a6eSPeter Krystad 	}
429cec37a6eSPeter Krystad 
4306d0060f6SMat Martineau 	timeo = sock_sndtimeo(sk, msg->msg_flags & MSG_DONTWAIT);
4316d0060f6SMat Martineau 
432cec37a6eSPeter Krystad 	ssk = mptcp_subflow_get(msk);
433cec37a6eSPeter Krystad 	if (!ssk) {
434cec37a6eSPeter Krystad 		release_sock(sk);
435cec37a6eSPeter Krystad 		return -ENOTCONN;
436cec37a6eSPeter Krystad 	}
437cec37a6eSPeter Krystad 
4386d0060f6SMat Martineau 	pr_debug("conn_list->subflow=%p", ssk);
439cec37a6eSPeter Krystad 
4406d0060f6SMat Martineau 	lock_sock(ssk);
4416d0060f6SMat Martineau 	while (msg_data_left(msg)) {
44257040755SPaolo Abeni 		ret = mptcp_sendmsg_frag(sk, ssk, msg, &timeo, &mss_now,
44357040755SPaolo Abeni 					 &size_goal);
4446d0060f6SMat Martineau 		if (ret < 0)
4456d0060f6SMat Martineau 			break;
4468ab183deSPaolo Abeni 		if (ret == 0 && unlikely(__mptcp_needs_tcp_fallback(msk))) {
4478ab183deSPaolo Abeni 			release_sock(ssk);
4488ab183deSPaolo Abeni 			ssock = __mptcp_tcp_fallback(msk);
4498ab183deSPaolo Abeni 			goto fallback;
4508ab183deSPaolo Abeni 		}
4516d0060f6SMat Martineau 
4526d0060f6SMat Martineau 		copied += ret;
4536d0060f6SMat Martineau 	}
4546d0060f6SMat Martineau 
45557040755SPaolo Abeni 	if (copied) {
4566d0060f6SMat Martineau 		ret = copied;
45757040755SPaolo Abeni 		tcp_push(ssk, msg->msg_flags, mss_now, tcp_sk(ssk)->nonagle,
45857040755SPaolo Abeni 			 size_goal);
45957040755SPaolo Abeni 	}
4606d0060f6SMat Martineau 
4611891c4a0SFlorian Westphal 	ssk_check_wmem(msk, ssk);
4626d0060f6SMat Martineau 	release_sock(ssk);
463cec37a6eSPeter Krystad 	release_sock(sk);
464cec37a6eSPeter Krystad 	return ret;
465f870fa0bSMat Martineau }
466f870fa0bSMat Martineau 
4677a6a6cbcSPaolo Abeni static void mptcp_wait_data(struct sock *sk, long *timeo)
4687a6a6cbcSPaolo Abeni {
4697a6a6cbcSPaolo Abeni 	DEFINE_WAIT_FUNC(wait, woken_wake_function);
4707a6a6cbcSPaolo Abeni 	struct mptcp_sock *msk = mptcp_sk(sk);
4717a6a6cbcSPaolo Abeni 
4727a6a6cbcSPaolo Abeni 	add_wait_queue(sk_sleep(sk), &wait);
4737a6a6cbcSPaolo Abeni 	sk_set_bit(SOCKWQ_ASYNC_WAITDATA, sk);
4747a6a6cbcSPaolo Abeni 
4757a6a6cbcSPaolo Abeni 	sk_wait_event(sk, timeo,
4767a6a6cbcSPaolo Abeni 		      test_and_clear_bit(MPTCP_DATA_READY, &msk->flags), &wait);
4777a6a6cbcSPaolo Abeni 
4787a6a6cbcSPaolo Abeni 	sk_clear_bit(SOCKWQ_ASYNC_WAITDATA, sk);
4797a6a6cbcSPaolo Abeni 	remove_wait_queue(sk_sleep(sk), &wait);
4807a6a6cbcSPaolo Abeni }
4817a6a6cbcSPaolo Abeni 
4826771bfd9SFlorian Westphal static int __mptcp_recvmsg_mskq(struct mptcp_sock *msk,
4836771bfd9SFlorian Westphal 				struct msghdr *msg,
4846771bfd9SFlorian Westphal 				size_t len)
4856771bfd9SFlorian Westphal {
4866771bfd9SFlorian Westphal 	struct sock *sk = (struct sock *)msk;
4876771bfd9SFlorian Westphal 	struct sk_buff *skb;
4886771bfd9SFlorian Westphal 	int copied = 0;
4896771bfd9SFlorian Westphal 
4906771bfd9SFlorian Westphal 	while ((skb = skb_peek(&sk->sk_receive_queue)) != NULL) {
4916771bfd9SFlorian Westphal 		u32 offset = MPTCP_SKB_CB(skb)->offset;
4926771bfd9SFlorian Westphal 		u32 data_len = skb->len - offset;
4936771bfd9SFlorian Westphal 		u32 count = min_t(size_t, len - copied, data_len);
4946771bfd9SFlorian Westphal 		int err;
4956771bfd9SFlorian Westphal 
4966771bfd9SFlorian Westphal 		err = skb_copy_datagram_msg(skb, offset, msg, count);
4976771bfd9SFlorian Westphal 		if (unlikely(err < 0)) {
4986771bfd9SFlorian Westphal 			if (!copied)
4996771bfd9SFlorian Westphal 				return err;
5006771bfd9SFlorian Westphal 			break;
5016771bfd9SFlorian Westphal 		}
5026771bfd9SFlorian Westphal 
5036771bfd9SFlorian Westphal 		copied += count;
5046771bfd9SFlorian Westphal 
5056771bfd9SFlorian Westphal 		if (count < data_len) {
5066771bfd9SFlorian Westphal 			MPTCP_SKB_CB(skb)->offset += count;
5076771bfd9SFlorian Westphal 			break;
5086771bfd9SFlorian Westphal 		}
5096771bfd9SFlorian Westphal 
5106771bfd9SFlorian Westphal 		__skb_unlink(skb, &sk->sk_receive_queue);
5116771bfd9SFlorian Westphal 		__kfree_skb(skb);
5126771bfd9SFlorian Westphal 
5136771bfd9SFlorian Westphal 		if (copied >= len)
5146771bfd9SFlorian Westphal 			break;
5156771bfd9SFlorian Westphal 	}
5166771bfd9SFlorian Westphal 
5176771bfd9SFlorian Westphal 	return copied;
5186771bfd9SFlorian Westphal }
5196771bfd9SFlorian Westphal 
5206771bfd9SFlorian Westphal static bool __mptcp_move_skbs(struct mptcp_sock *msk)
5216771bfd9SFlorian Westphal {
5226771bfd9SFlorian Westphal 	unsigned int moved = 0;
5236771bfd9SFlorian Westphal 	bool done;
5246771bfd9SFlorian Westphal 
5256771bfd9SFlorian Westphal 	do {
5266771bfd9SFlorian Westphal 		struct sock *ssk = mptcp_subflow_recv_lookup(msk);
5276771bfd9SFlorian Westphal 
5286771bfd9SFlorian Westphal 		if (!ssk)
5296771bfd9SFlorian Westphal 			break;
5306771bfd9SFlorian Westphal 
5316771bfd9SFlorian Westphal 		lock_sock(ssk);
5326771bfd9SFlorian Westphal 		done = __mptcp_move_skbs_from_subflow(msk, ssk, &moved);
5336771bfd9SFlorian Westphal 		release_sock(ssk);
5346771bfd9SFlorian Westphal 	} while (!done);
5356771bfd9SFlorian Westphal 
5366771bfd9SFlorian Westphal 	return moved > 0;
5376771bfd9SFlorian Westphal }
5386771bfd9SFlorian Westphal 
539f870fa0bSMat Martineau static int mptcp_recvmsg(struct sock *sk, struct msghdr *msg, size_t len,
540f870fa0bSMat Martineau 			 int nonblock, int flags, int *addr_len)
541f870fa0bSMat Martineau {
542f870fa0bSMat Martineau 	struct mptcp_sock *msk = mptcp_sk(sk);
543cec37a6eSPeter Krystad 	struct socket *ssock;
544cec37a6eSPeter Krystad 	int copied = 0;
5457a6a6cbcSPaolo Abeni 	int target;
5467a6a6cbcSPaolo Abeni 	long timeo;
547f870fa0bSMat Martineau 
548f870fa0bSMat Martineau 	if (msg->msg_flags & ~(MSG_WAITALL | MSG_DONTWAIT))
549f870fa0bSMat Martineau 		return -EOPNOTSUPP;
550f870fa0bSMat Martineau 
551cec37a6eSPeter Krystad 	lock_sock(sk);
552cec37a6eSPeter Krystad 	ssock = __mptcp_tcp_fallback(msk);
5538ab183deSPaolo Abeni 	if (unlikely(ssock)) {
5548ab183deSPaolo Abeni fallback:
555cec37a6eSPeter Krystad 		pr_debug("fallback-read subflow=%p",
556cec37a6eSPeter Krystad 			 mptcp_subflow_ctx(ssock->sk));
557cec37a6eSPeter Krystad 		copied = sock_recvmsg(ssock, msg, flags);
558cec37a6eSPeter Krystad 		return copied;
559cec37a6eSPeter Krystad 	}
560cec37a6eSPeter Krystad 
5617a6a6cbcSPaolo Abeni 	timeo = sock_rcvtimeo(sk, nonblock);
5627a6a6cbcSPaolo Abeni 
5637a6a6cbcSPaolo Abeni 	len = min_t(size_t, len, INT_MAX);
5647a6a6cbcSPaolo Abeni 	target = sock_rcvlowat(sk, flags & MSG_WAITALL, len);
5657a6a6cbcSPaolo Abeni 
5666771bfd9SFlorian Westphal 	while (len > (size_t)copied) {
5677a6a6cbcSPaolo Abeni 		int bytes_read;
5687a6a6cbcSPaolo Abeni 
5696771bfd9SFlorian Westphal 		bytes_read = __mptcp_recvmsg_mskq(msk, msg, len - copied);
5706771bfd9SFlorian Westphal 		if (unlikely(bytes_read < 0)) {
5717a6a6cbcSPaolo Abeni 			if (!copied)
5727a6a6cbcSPaolo Abeni 				copied = bytes_read;
5736771bfd9SFlorian Westphal 			goto out_err;
574cec37a6eSPeter Krystad 		}
575cec37a6eSPeter Krystad 
5767a6a6cbcSPaolo Abeni 		copied += bytes_read;
5777a6a6cbcSPaolo Abeni 
5786771bfd9SFlorian Westphal 		if (skb_queue_empty(&sk->sk_receive_queue) &&
5796771bfd9SFlorian Westphal 		    __mptcp_move_skbs(msk))
5806771bfd9SFlorian Westphal 			continue;
5817a6a6cbcSPaolo Abeni 
5827a6a6cbcSPaolo Abeni 		/* only the master socket status is relevant here. The exit
5837a6a6cbcSPaolo Abeni 		 * conditions mirror closely tcp_recvmsg()
5847a6a6cbcSPaolo Abeni 		 */
5857a6a6cbcSPaolo Abeni 		if (copied >= target)
5867a6a6cbcSPaolo Abeni 			break;
5877a6a6cbcSPaolo Abeni 
5887a6a6cbcSPaolo Abeni 		if (copied) {
5897a6a6cbcSPaolo Abeni 			if (sk->sk_err ||
5907a6a6cbcSPaolo Abeni 			    sk->sk_state == TCP_CLOSE ||
5917a6a6cbcSPaolo Abeni 			    (sk->sk_shutdown & RCV_SHUTDOWN) ||
5927a6a6cbcSPaolo Abeni 			    !timeo ||
5937a6a6cbcSPaolo Abeni 			    signal_pending(current))
5947a6a6cbcSPaolo Abeni 				break;
5957a6a6cbcSPaolo Abeni 		} else {
5967a6a6cbcSPaolo Abeni 			if (sk->sk_err) {
5977a6a6cbcSPaolo Abeni 				copied = sock_error(sk);
5987a6a6cbcSPaolo Abeni 				break;
5997a6a6cbcSPaolo Abeni 			}
6007a6a6cbcSPaolo Abeni 
6017a6a6cbcSPaolo Abeni 			if (sk->sk_shutdown & RCV_SHUTDOWN)
6027a6a6cbcSPaolo Abeni 				break;
6037a6a6cbcSPaolo Abeni 
6047a6a6cbcSPaolo Abeni 			if (sk->sk_state == TCP_CLOSE) {
6057a6a6cbcSPaolo Abeni 				copied = -ENOTCONN;
6067a6a6cbcSPaolo Abeni 				break;
6077a6a6cbcSPaolo Abeni 			}
6087a6a6cbcSPaolo Abeni 
6097a6a6cbcSPaolo Abeni 			if (!timeo) {
6107a6a6cbcSPaolo Abeni 				copied = -EAGAIN;
6117a6a6cbcSPaolo Abeni 				break;
6127a6a6cbcSPaolo Abeni 			}
6137a6a6cbcSPaolo Abeni 
6147a6a6cbcSPaolo Abeni 			if (signal_pending(current)) {
6157a6a6cbcSPaolo Abeni 				copied = sock_intr_errno(timeo);
6167a6a6cbcSPaolo Abeni 				break;
6177a6a6cbcSPaolo Abeni 			}
6187a6a6cbcSPaolo Abeni 		}
6197a6a6cbcSPaolo Abeni 
6207a6a6cbcSPaolo Abeni 		pr_debug("block timeout %ld", timeo);
6217a6a6cbcSPaolo Abeni 		mptcp_wait_data(sk, &timeo);
6228ab183deSPaolo Abeni 		if (unlikely(__mptcp_tcp_fallback(msk)))
6238ab183deSPaolo Abeni 			goto fallback;
6247a6a6cbcSPaolo Abeni 	}
6257a6a6cbcSPaolo Abeni 
6266771bfd9SFlorian Westphal 	if (skb_queue_empty(&sk->sk_receive_queue)) {
6276771bfd9SFlorian Westphal 		/* entire backlog drained, clear DATA_READY. */
6287a6a6cbcSPaolo Abeni 		clear_bit(MPTCP_DATA_READY, &msk->flags);
6297a6a6cbcSPaolo Abeni 
6306771bfd9SFlorian Westphal 		/* .. race-breaker: ssk might have gotten new data
6316771bfd9SFlorian Westphal 		 * after last __mptcp_move_skbs() returned false.
6327a6a6cbcSPaolo Abeni 		 */
6336771bfd9SFlorian Westphal 		if (unlikely(__mptcp_move_skbs(msk)))
6346771bfd9SFlorian Westphal 			set_bit(MPTCP_DATA_READY, &msk->flags);
6356771bfd9SFlorian Westphal 	} else if (unlikely(!test_bit(MPTCP_DATA_READY, &msk->flags))) {
6366771bfd9SFlorian Westphal 		/* data to read but mptcp_wait_data() cleared DATA_READY */
6377a6a6cbcSPaolo Abeni 		set_bit(MPTCP_DATA_READY, &msk->flags);
6387a6a6cbcSPaolo Abeni 	}
6396771bfd9SFlorian Westphal out_err:
640cec37a6eSPeter Krystad 	release_sock(sk);
641cec37a6eSPeter Krystad 	return copied;
642cec37a6eSPeter Krystad }
643cec37a6eSPeter Krystad 
644cec37a6eSPeter Krystad /* subflow sockets can be either outgoing (connect) or incoming
645cec37a6eSPeter Krystad  * (accept).
646cec37a6eSPeter Krystad  *
647cec37a6eSPeter Krystad  * Outgoing subflows use in-kernel sockets.
648cec37a6eSPeter Krystad  * Incoming subflows do not have their own 'struct socket' allocated,
649cec37a6eSPeter Krystad  * so we need to use tcp_close() after detaching them from the mptcp
650cec37a6eSPeter Krystad  * parent socket.
651cec37a6eSPeter Krystad  */
652cec37a6eSPeter Krystad static void __mptcp_close_ssk(struct sock *sk, struct sock *ssk,
653cec37a6eSPeter Krystad 			      struct mptcp_subflow_context *subflow,
654cec37a6eSPeter Krystad 			      long timeout)
655cec37a6eSPeter Krystad {
656cec37a6eSPeter Krystad 	struct socket *sock = READ_ONCE(ssk->sk_socket);
657cec37a6eSPeter Krystad 
658cec37a6eSPeter Krystad 	list_del(&subflow->node);
659cec37a6eSPeter Krystad 
660cec37a6eSPeter Krystad 	if (sock && sock != sk->sk_socket) {
661cec37a6eSPeter Krystad 		/* outgoing subflow */
662cec37a6eSPeter Krystad 		sock_release(sock);
663cec37a6eSPeter Krystad 	} else {
664cec37a6eSPeter Krystad 		/* incoming subflow */
665cec37a6eSPeter Krystad 		tcp_close(ssk, timeout);
666cec37a6eSPeter Krystad 	}
667f870fa0bSMat Martineau }
668f870fa0bSMat Martineau 
66980992017SPaolo Abeni static void mptcp_worker(struct work_struct *work)
67080992017SPaolo Abeni {
67180992017SPaolo Abeni 	struct mptcp_sock *msk = container_of(work, struct mptcp_sock, work);
67280992017SPaolo Abeni 	struct sock *sk = &msk->sk.icsk_inet.sk;
67380992017SPaolo Abeni 
67480992017SPaolo Abeni 	lock_sock(sk);
6756771bfd9SFlorian Westphal 	__mptcp_move_skbs(msk);
67680992017SPaolo Abeni 	release_sock(sk);
67780992017SPaolo Abeni 	sock_put(sk);
67880992017SPaolo Abeni }
67980992017SPaolo Abeni 
680784325e9SMatthieu Baerts static int __mptcp_init_sock(struct sock *sk)
681f870fa0bSMat Martineau {
682cec37a6eSPeter Krystad 	struct mptcp_sock *msk = mptcp_sk(sk);
683cec37a6eSPeter Krystad 
684cec37a6eSPeter Krystad 	INIT_LIST_HEAD(&msk->conn_list);
6851891c4a0SFlorian Westphal 	__set_bit(MPTCP_SEND_SPACE, &msk->flags);
68680992017SPaolo Abeni 	INIT_WORK(&msk->work, mptcp_worker);
687cec37a6eSPeter Krystad 
6888ab183deSPaolo Abeni 	msk->first = NULL;
6898ab183deSPaolo Abeni 
690f870fa0bSMat Martineau 	return 0;
691f870fa0bSMat Martineau }
692f870fa0bSMat Martineau 
693784325e9SMatthieu Baerts static int mptcp_init_sock(struct sock *sk)
694784325e9SMatthieu Baerts {
695784325e9SMatthieu Baerts 	if (!mptcp_is_enabled(sock_net(sk)))
696784325e9SMatthieu Baerts 		return -ENOPROTOOPT;
697784325e9SMatthieu Baerts 
698784325e9SMatthieu Baerts 	return __mptcp_init_sock(sk);
699784325e9SMatthieu Baerts }
700784325e9SMatthieu Baerts 
70180992017SPaolo Abeni static void mptcp_cancel_work(struct sock *sk)
70280992017SPaolo Abeni {
70380992017SPaolo Abeni 	struct mptcp_sock *msk = mptcp_sk(sk);
70480992017SPaolo Abeni 
70580992017SPaolo Abeni 	if (cancel_work_sync(&msk->work))
70680992017SPaolo Abeni 		sock_put(sk);
70780992017SPaolo Abeni }
70880992017SPaolo Abeni 
70921498490SPeter Krystad static void mptcp_subflow_shutdown(struct sock *ssk, int how)
71021498490SPeter Krystad {
71121498490SPeter Krystad 	lock_sock(ssk);
71221498490SPeter Krystad 
71321498490SPeter Krystad 	switch (ssk->sk_state) {
71421498490SPeter Krystad 	case TCP_LISTEN:
71521498490SPeter Krystad 		if (!(how & RCV_SHUTDOWN))
71621498490SPeter Krystad 			break;
71721498490SPeter Krystad 		/* fall through */
71821498490SPeter Krystad 	case TCP_SYN_SENT:
71921498490SPeter Krystad 		tcp_disconnect(ssk, O_NONBLOCK);
72021498490SPeter Krystad 		break;
72121498490SPeter Krystad 	default:
72221498490SPeter Krystad 		ssk->sk_shutdown |= how;
72321498490SPeter Krystad 		tcp_shutdown(ssk, how);
72421498490SPeter Krystad 		break;
72521498490SPeter Krystad 	}
72621498490SPeter Krystad 
72721498490SPeter Krystad 	/* Wake up anyone sleeping in poll. */
72821498490SPeter Krystad 	ssk->sk_state_change(ssk);
72921498490SPeter Krystad 	release_sock(ssk);
73021498490SPeter Krystad }
73121498490SPeter Krystad 
7328ab183deSPaolo Abeni /* Called with msk lock held, releases such lock before returning */
7332c22c06cSFlorian Westphal static void mptcp_close(struct sock *sk, long timeout)
734f870fa0bSMat Martineau {
735cec37a6eSPeter Krystad 	struct mptcp_subflow_context *subflow, *tmp;
736f870fa0bSMat Martineau 	struct mptcp_sock *msk = mptcp_sk(sk);
737b2c5b614SFlorian Westphal 	LIST_HEAD(conn_list);
738f870fa0bSMat Martineau 
7392c22c06cSFlorian Westphal 	lock_sock(sk);
7402c22c06cSFlorian Westphal 
74179c0949eSPeter Krystad 	mptcp_token_destroy(msk->token);
742f870fa0bSMat Martineau 	inet_sk_state_store(sk, TCP_CLOSE);
743f870fa0bSMat Martineau 
744b2c5b614SFlorian Westphal 	list_splice_init(&msk->conn_list, &conn_list);
745b2c5b614SFlorian Westphal 
746b2c5b614SFlorian Westphal 	release_sock(sk);
747b2c5b614SFlorian Westphal 
748b2c5b614SFlorian Westphal 	list_for_each_entry_safe(subflow, tmp, &conn_list, node) {
749cec37a6eSPeter Krystad 		struct sock *ssk = mptcp_subflow_tcp_sock(subflow);
750cec37a6eSPeter Krystad 
751cec37a6eSPeter Krystad 		__mptcp_close_ssk(sk, ssk, subflow, timeout);
752f870fa0bSMat Martineau 	}
753f870fa0bSMat Martineau 
75480992017SPaolo Abeni 	mptcp_cancel_work(sk);
75580992017SPaolo Abeni 
7566771bfd9SFlorian Westphal 	__skb_queue_purge(&sk->sk_receive_queue);
7576771bfd9SFlorian Westphal 
758cec37a6eSPeter Krystad 	sk_common_release(sk);
759f870fa0bSMat Martineau }
760f870fa0bSMat Martineau 
761cf7da0d6SPeter Krystad static void mptcp_copy_inaddrs(struct sock *msk, const struct sock *ssk)
762cf7da0d6SPeter Krystad {
763cf7da0d6SPeter Krystad #if IS_ENABLED(CONFIG_MPTCP_IPV6)
764cf7da0d6SPeter Krystad 	const struct ipv6_pinfo *ssk6 = inet6_sk(ssk);
765cf7da0d6SPeter Krystad 	struct ipv6_pinfo *msk6 = inet6_sk(msk);
766cf7da0d6SPeter Krystad 
767cf7da0d6SPeter Krystad 	msk->sk_v6_daddr = ssk->sk_v6_daddr;
768cf7da0d6SPeter Krystad 	msk->sk_v6_rcv_saddr = ssk->sk_v6_rcv_saddr;
769cf7da0d6SPeter Krystad 
770cf7da0d6SPeter Krystad 	if (msk6 && ssk6) {
771cf7da0d6SPeter Krystad 		msk6->saddr = ssk6->saddr;
772cf7da0d6SPeter Krystad 		msk6->flow_label = ssk6->flow_label;
773cf7da0d6SPeter Krystad 	}
774cf7da0d6SPeter Krystad #endif
775cf7da0d6SPeter Krystad 
776cf7da0d6SPeter Krystad 	inet_sk(msk)->inet_num = inet_sk(ssk)->inet_num;
777cf7da0d6SPeter Krystad 	inet_sk(msk)->inet_dport = inet_sk(ssk)->inet_dport;
778cf7da0d6SPeter Krystad 	inet_sk(msk)->inet_sport = inet_sk(ssk)->inet_sport;
779cf7da0d6SPeter Krystad 	inet_sk(msk)->inet_daddr = inet_sk(ssk)->inet_daddr;
780cf7da0d6SPeter Krystad 	inet_sk(msk)->inet_saddr = inet_sk(ssk)->inet_saddr;
781cf7da0d6SPeter Krystad 	inet_sk(msk)->inet_rcv_saddr = inet_sk(ssk)->inet_rcv_saddr;
782cf7da0d6SPeter Krystad }
783cf7da0d6SPeter Krystad 
784b0519de8SFlorian Westphal #if IS_ENABLED(CONFIG_MPTCP_IPV6)
785b0519de8SFlorian Westphal static struct ipv6_pinfo *mptcp_inet6_sk(const struct sock *sk)
786b0519de8SFlorian Westphal {
787b0519de8SFlorian Westphal 	unsigned int offset = sizeof(struct mptcp6_sock) - sizeof(struct ipv6_pinfo);
788b0519de8SFlorian Westphal 
789b0519de8SFlorian Westphal 	return (struct ipv6_pinfo *)(((u8 *)sk) + offset);
790b0519de8SFlorian Westphal }
791b0519de8SFlorian Westphal #endif
792b0519de8SFlorian Westphal 
7935609e2bbSChen Wandun static struct sock *mptcp_sk_clone_lock(const struct sock *sk)
794b0519de8SFlorian Westphal {
795b0519de8SFlorian Westphal 	struct sock *nsk = sk_clone_lock(sk, GFP_ATOMIC);
796b0519de8SFlorian Westphal 
797b0519de8SFlorian Westphal 	if (!nsk)
798b0519de8SFlorian Westphal 		return NULL;
799b0519de8SFlorian Westphal 
800b0519de8SFlorian Westphal #if IS_ENABLED(CONFIG_MPTCP_IPV6)
801b0519de8SFlorian Westphal 	if (nsk->sk_family == AF_INET6)
802b0519de8SFlorian Westphal 		inet_sk(nsk)->pinet6 = mptcp_inet6_sk(nsk);
803b0519de8SFlorian Westphal #endif
804b0519de8SFlorian Westphal 
805b0519de8SFlorian Westphal 	return nsk;
806b0519de8SFlorian Westphal }
807b0519de8SFlorian Westphal 
808cf7da0d6SPeter Krystad static struct sock *mptcp_accept(struct sock *sk, int flags, int *err,
809cf7da0d6SPeter Krystad 				 bool kern)
810cf7da0d6SPeter Krystad {
811cf7da0d6SPeter Krystad 	struct mptcp_sock *msk = mptcp_sk(sk);
812cf7da0d6SPeter Krystad 	struct socket *listener;
813cf7da0d6SPeter Krystad 	struct sock *newsk;
814cf7da0d6SPeter Krystad 
815cf7da0d6SPeter Krystad 	listener = __mptcp_nmpc_socket(msk);
816cf7da0d6SPeter Krystad 	if (WARN_ON_ONCE(!listener)) {
817cf7da0d6SPeter Krystad 		*err = -EINVAL;
818cf7da0d6SPeter Krystad 		return NULL;
819cf7da0d6SPeter Krystad 	}
820cf7da0d6SPeter Krystad 
821cf7da0d6SPeter Krystad 	pr_debug("msk=%p, listener=%p", msk, mptcp_subflow_ctx(listener->sk));
822cf7da0d6SPeter Krystad 	newsk = inet_csk_accept(listener->sk, flags, err, kern);
823cf7da0d6SPeter Krystad 	if (!newsk)
824cf7da0d6SPeter Krystad 		return NULL;
825cf7da0d6SPeter Krystad 
826cf7da0d6SPeter Krystad 	pr_debug("msk=%p, subflow is mptcp=%d", msk, sk_is_mptcp(newsk));
827cf7da0d6SPeter Krystad 
828cf7da0d6SPeter Krystad 	if (sk_is_mptcp(newsk)) {
829cf7da0d6SPeter Krystad 		struct mptcp_subflow_context *subflow;
830cf7da0d6SPeter Krystad 		struct sock *new_mptcp_sock;
831cf7da0d6SPeter Krystad 		struct sock *ssk = newsk;
8326d0060f6SMat Martineau 		u64 ack_seq;
833cf7da0d6SPeter Krystad 
834cf7da0d6SPeter Krystad 		subflow = mptcp_subflow_ctx(newsk);
835cf7da0d6SPeter Krystad 		lock_sock(sk);
836cf7da0d6SPeter Krystad 
837cf7da0d6SPeter Krystad 		local_bh_disable();
838b0519de8SFlorian Westphal 		new_mptcp_sock = mptcp_sk_clone_lock(sk);
839cf7da0d6SPeter Krystad 		if (!new_mptcp_sock) {
840cf7da0d6SPeter Krystad 			*err = -ENOBUFS;
841cf7da0d6SPeter Krystad 			local_bh_enable();
842cf7da0d6SPeter Krystad 			release_sock(sk);
84321498490SPeter Krystad 			mptcp_subflow_shutdown(newsk, SHUT_RDWR + 1);
844cf7da0d6SPeter Krystad 			tcp_close(newsk, 0);
845cf7da0d6SPeter Krystad 			return NULL;
846cf7da0d6SPeter Krystad 		}
847cf7da0d6SPeter Krystad 
848784325e9SMatthieu Baerts 		__mptcp_init_sock(new_mptcp_sock);
849cf7da0d6SPeter Krystad 
850cf7da0d6SPeter Krystad 		msk = mptcp_sk(new_mptcp_sock);
851cf7da0d6SPeter Krystad 		msk->local_key = subflow->local_key;
85279c0949eSPeter Krystad 		msk->token = subflow->token;
853cf7da0d6SPeter Krystad 		msk->subflow = NULL;
8548ab183deSPaolo Abeni 		msk->first = newsk;
855cf7da0d6SPeter Krystad 
85679c0949eSPeter Krystad 		mptcp_token_update_accept(newsk, new_mptcp_sock);
8576d0060f6SMat Martineau 
8586d0060f6SMat Martineau 		msk->write_seq = subflow->idsn + 1;
859d22f4988SChristoph Paasch 		if (subflow->can_ack) {
860d22f4988SChristoph Paasch 			msk->can_ack = true;
861d22f4988SChristoph Paasch 			msk->remote_key = subflow->remote_key;
862d22f4988SChristoph Paasch 			mptcp_crypto_key_sha(msk->remote_key, NULL, &ack_seq);
8636d0060f6SMat Martineau 			ack_seq++;
8646d0060f6SMat Martineau 			msk->ack_seq = ack_seq;
865d22f4988SChristoph Paasch 		}
866cf7da0d6SPeter Krystad 		newsk = new_mptcp_sock;
867cf7da0d6SPeter Krystad 		mptcp_copy_inaddrs(newsk, ssk);
868cf7da0d6SPeter Krystad 		list_add(&subflow->node, &msk->conn_list);
869cf7da0d6SPeter Krystad 
870cf7da0d6SPeter Krystad 		/* will be fully established at mptcp_stream_accept()
871cf7da0d6SPeter Krystad 		 * completion.
872cf7da0d6SPeter Krystad 		 */
873cf7da0d6SPeter Krystad 		inet_sk_state_store(new_mptcp_sock, TCP_SYN_RECV);
874cf7da0d6SPeter Krystad 		bh_unlock_sock(new_mptcp_sock);
875cf7da0d6SPeter Krystad 		local_bh_enable();
876cf7da0d6SPeter Krystad 		release_sock(sk);
8777a6a6cbcSPaolo Abeni 
8787a6a6cbcSPaolo Abeni 		/* the subflow can already receive packet, avoid racing with
8797a6a6cbcSPaolo Abeni 		 * the receive path and process the pending ones
8807a6a6cbcSPaolo Abeni 		 */
8817a6a6cbcSPaolo Abeni 		lock_sock(ssk);
8827a6a6cbcSPaolo Abeni 		subflow->rel_write_seq = 1;
8837a6a6cbcSPaolo Abeni 		subflow->tcp_sock = ssk;
8847a6a6cbcSPaolo Abeni 		subflow->conn = new_mptcp_sock;
8857a6a6cbcSPaolo Abeni 		if (unlikely(!skb_queue_empty(&ssk->sk_receive_queue)))
8867a6a6cbcSPaolo Abeni 			mptcp_subflow_data_available(ssk);
8877a6a6cbcSPaolo Abeni 		release_sock(ssk);
888cf7da0d6SPeter Krystad 	}
889cf7da0d6SPeter Krystad 
890cf7da0d6SPeter Krystad 	return newsk;
891cf7da0d6SPeter Krystad }
892cf7da0d6SPeter Krystad 
89379c0949eSPeter Krystad static void mptcp_destroy(struct sock *sk)
89479c0949eSPeter Krystad {
895c9fd9c5fSFlorian Westphal 	struct mptcp_sock *msk = mptcp_sk(sk);
896c9fd9c5fSFlorian Westphal 
897c9fd9c5fSFlorian Westphal 	if (msk->cached_ext)
898c9fd9c5fSFlorian Westphal 		__skb_ext_put(msk->cached_ext);
89979c0949eSPeter Krystad }
90079c0949eSPeter Krystad 
901717e79c8SPeter Krystad static int mptcp_setsockopt(struct sock *sk, int level, int optname,
90250e741bbSFlorian Westphal 			    char __user *optval, unsigned int optlen)
903717e79c8SPeter Krystad {
904717e79c8SPeter Krystad 	struct mptcp_sock *msk = mptcp_sk(sk);
905717e79c8SPeter Krystad 	struct socket *ssock;
906717e79c8SPeter Krystad 
907717e79c8SPeter Krystad 	pr_debug("msk=%p", msk);
908717e79c8SPeter Krystad 
909717e79c8SPeter Krystad 	/* @@ the meaning of setsockopt() when the socket is connected and
910b6e4a1aeSMat Martineau 	 * there are multiple subflows is not yet defined. It is up to the
911b6e4a1aeSMat Martineau 	 * MPTCP-level socket to configure the subflows until the subflow
912b6e4a1aeSMat Martineau 	 * is in TCP fallback, when TCP socket options are passed through
913b6e4a1aeSMat Martineau 	 * to the one remaining subflow.
914717e79c8SPeter Krystad 	 */
915717e79c8SPeter Krystad 	lock_sock(sk);
916b6e4a1aeSMat Martineau 	ssock = __mptcp_tcp_fallback(msk);
917b6e4a1aeSMat Martineau 	if (ssock)
918b6e4a1aeSMat Martineau 		return tcp_setsockopt(ssock->sk, level, optname, optval,
919b6e4a1aeSMat Martineau 				      optlen);
92050e741bbSFlorian Westphal 
92150e741bbSFlorian Westphal 	release_sock(sk);
92250e741bbSFlorian Westphal 
923b6e4a1aeSMat Martineau 	return -EOPNOTSUPP;
924717e79c8SPeter Krystad }
925717e79c8SPeter Krystad 
926717e79c8SPeter Krystad static int mptcp_getsockopt(struct sock *sk, int level, int optname,
92750e741bbSFlorian Westphal 			    char __user *optval, int __user *option)
928717e79c8SPeter Krystad {
929717e79c8SPeter Krystad 	struct mptcp_sock *msk = mptcp_sk(sk);
930717e79c8SPeter Krystad 	struct socket *ssock;
931717e79c8SPeter Krystad 
932717e79c8SPeter Krystad 	pr_debug("msk=%p", msk);
933717e79c8SPeter Krystad 
934b6e4a1aeSMat Martineau 	/* @@ the meaning of setsockopt() when the socket is connected and
935b6e4a1aeSMat Martineau 	 * there are multiple subflows is not yet defined. It is up to the
936b6e4a1aeSMat Martineau 	 * MPTCP-level socket to configure the subflows until the subflow
937b6e4a1aeSMat Martineau 	 * is in TCP fallback, when socket options are passed through
938b6e4a1aeSMat Martineau 	 * to the one remaining subflow.
939717e79c8SPeter Krystad 	 */
940717e79c8SPeter Krystad 	lock_sock(sk);
941b6e4a1aeSMat Martineau 	ssock = __mptcp_tcp_fallback(msk);
942b6e4a1aeSMat Martineau 	if (ssock)
943b6e4a1aeSMat Martineau 		return tcp_getsockopt(ssock->sk, level, optname, optval,
944b6e4a1aeSMat Martineau 				      option);
94550e741bbSFlorian Westphal 
94650e741bbSFlorian Westphal 	release_sock(sk);
94750e741bbSFlorian Westphal 
948b6e4a1aeSMat Martineau 	return -EOPNOTSUPP;
949717e79c8SPeter Krystad }
950717e79c8SPeter Krystad 
951*14c441b5SPaolo Abeni #define MPTCP_DEFERRED_ALL TCPF_DELACK_TIMER_DEFERRED
952*14c441b5SPaolo Abeni 
953*14c441b5SPaolo Abeni /* this is very alike tcp_release_cb() but we must handle differently a
954*14c441b5SPaolo Abeni  * different set of events
955*14c441b5SPaolo Abeni  */
956*14c441b5SPaolo Abeni static void mptcp_release_cb(struct sock *sk)
957*14c441b5SPaolo Abeni {
958*14c441b5SPaolo Abeni 	unsigned long flags, nflags;
959*14c441b5SPaolo Abeni 
960*14c441b5SPaolo Abeni 	do {
961*14c441b5SPaolo Abeni 		flags = sk->sk_tsq_flags;
962*14c441b5SPaolo Abeni 		if (!(flags & MPTCP_DEFERRED_ALL))
963*14c441b5SPaolo Abeni 			return;
964*14c441b5SPaolo Abeni 		nflags = flags & ~MPTCP_DEFERRED_ALL;
965*14c441b5SPaolo Abeni 	} while (cmpxchg(&sk->sk_tsq_flags, flags, nflags) != flags);
966*14c441b5SPaolo Abeni 
967*14c441b5SPaolo Abeni 	if (flags & TCPF_DELACK_TIMER_DEFERRED) {
968*14c441b5SPaolo Abeni 		struct mptcp_sock *msk = mptcp_sk(sk);
969*14c441b5SPaolo Abeni 		struct sock *ssk;
970*14c441b5SPaolo Abeni 
971*14c441b5SPaolo Abeni 		ssk = mptcp_subflow_recv_lookup(msk);
972*14c441b5SPaolo Abeni 		if (!ssk || !schedule_work(&msk->work))
973*14c441b5SPaolo Abeni 			__sock_put(sk);
974*14c441b5SPaolo Abeni 	}
975*14c441b5SPaolo Abeni }
976*14c441b5SPaolo Abeni 
977cec37a6eSPeter Krystad static int mptcp_get_port(struct sock *sk, unsigned short snum)
978f870fa0bSMat Martineau {
979f870fa0bSMat Martineau 	struct mptcp_sock *msk = mptcp_sk(sk);
980cec37a6eSPeter Krystad 	struct socket *ssock;
981f870fa0bSMat Martineau 
982cec37a6eSPeter Krystad 	ssock = __mptcp_nmpc_socket(msk);
983cec37a6eSPeter Krystad 	pr_debug("msk=%p, subflow=%p", msk, ssock);
984cec37a6eSPeter Krystad 	if (WARN_ON_ONCE(!ssock))
985cec37a6eSPeter Krystad 		return -EINVAL;
986f870fa0bSMat Martineau 
987cec37a6eSPeter Krystad 	return inet_csk_get_port(ssock->sk, snum);
988cec37a6eSPeter Krystad }
989f870fa0bSMat Martineau 
990cec37a6eSPeter Krystad void mptcp_finish_connect(struct sock *ssk)
991cec37a6eSPeter Krystad {
992cec37a6eSPeter Krystad 	struct mptcp_subflow_context *subflow;
993cec37a6eSPeter Krystad 	struct mptcp_sock *msk;
994cec37a6eSPeter Krystad 	struct sock *sk;
9956d0060f6SMat Martineau 	u64 ack_seq;
996f870fa0bSMat Martineau 
997cec37a6eSPeter Krystad 	subflow = mptcp_subflow_ctx(ssk);
998f870fa0bSMat Martineau 
999cec37a6eSPeter Krystad 	if (!subflow->mp_capable)
1000cec37a6eSPeter Krystad 		return;
1001cec37a6eSPeter Krystad 
1002cec37a6eSPeter Krystad 	sk = subflow->conn;
1003cec37a6eSPeter Krystad 	msk = mptcp_sk(sk);
1004cec37a6eSPeter Krystad 
1005648ef4b8SMat Martineau 	pr_debug("msk=%p, token=%u", sk, subflow->token);
1006648ef4b8SMat Martineau 
10076d0060f6SMat Martineau 	mptcp_crypto_key_sha(subflow->remote_key, NULL, &ack_seq);
10086d0060f6SMat Martineau 	ack_seq++;
1009648ef4b8SMat Martineau 	subflow->map_seq = ack_seq;
1010648ef4b8SMat Martineau 	subflow->map_subflow_seq = 1;
10116d0060f6SMat Martineau 	subflow->rel_write_seq = 1;
10126d0060f6SMat Martineau 
1013cec37a6eSPeter Krystad 	/* the socket is not connected yet, no msk/subflow ops can access/race
1014cec37a6eSPeter Krystad 	 * accessing the field below
1015cec37a6eSPeter Krystad 	 */
1016cec37a6eSPeter Krystad 	WRITE_ONCE(msk->remote_key, subflow->remote_key);
1017cec37a6eSPeter Krystad 	WRITE_ONCE(msk->local_key, subflow->local_key);
101879c0949eSPeter Krystad 	WRITE_ONCE(msk->token, subflow->token);
10196d0060f6SMat Martineau 	WRITE_ONCE(msk->write_seq, subflow->idsn + 1);
10206d0060f6SMat Martineau 	WRITE_ONCE(msk->ack_seq, ack_seq);
1021d22f4988SChristoph Paasch 	WRITE_ONCE(msk->can_ack, 1);
1022f870fa0bSMat Martineau }
1023f870fa0bSMat Martineau 
1024cf7da0d6SPeter Krystad static void mptcp_sock_graft(struct sock *sk, struct socket *parent)
1025cf7da0d6SPeter Krystad {
1026cf7da0d6SPeter Krystad 	write_lock_bh(&sk->sk_callback_lock);
1027cf7da0d6SPeter Krystad 	rcu_assign_pointer(sk->sk_wq, &parent->wq);
1028cf7da0d6SPeter Krystad 	sk_set_socket(sk, parent);
1029cf7da0d6SPeter Krystad 	sk->sk_uid = SOCK_INODE(parent)->i_uid;
1030cf7da0d6SPeter Krystad 	write_unlock_bh(&sk->sk_callback_lock);
1031cf7da0d6SPeter Krystad }
1032cf7da0d6SPeter Krystad 
10331891c4a0SFlorian Westphal static bool mptcp_memory_free(const struct sock *sk, int wake)
10341891c4a0SFlorian Westphal {
10351891c4a0SFlorian Westphal 	struct mptcp_sock *msk = mptcp_sk(sk);
10361891c4a0SFlorian Westphal 
10371891c4a0SFlorian Westphal 	return wake ? test_bit(MPTCP_SEND_SPACE, &msk->flags) : true;
10381891c4a0SFlorian Westphal }
10391891c4a0SFlorian Westphal 
1040f870fa0bSMat Martineau static struct proto mptcp_prot = {
1041f870fa0bSMat Martineau 	.name		= "MPTCP",
1042f870fa0bSMat Martineau 	.owner		= THIS_MODULE,
1043f870fa0bSMat Martineau 	.init		= mptcp_init_sock,
1044f870fa0bSMat Martineau 	.close		= mptcp_close,
1045cf7da0d6SPeter Krystad 	.accept		= mptcp_accept,
1046717e79c8SPeter Krystad 	.setsockopt	= mptcp_setsockopt,
1047717e79c8SPeter Krystad 	.getsockopt	= mptcp_getsockopt,
1048f870fa0bSMat Martineau 	.shutdown	= tcp_shutdown,
104979c0949eSPeter Krystad 	.destroy	= mptcp_destroy,
1050f870fa0bSMat Martineau 	.sendmsg	= mptcp_sendmsg,
1051f870fa0bSMat Martineau 	.recvmsg	= mptcp_recvmsg,
1052*14c441b5SPaolo Abeni 	.release_cb	= mptcp_release_cb,
1053f870fa0bSMat Martineau 	.hash		= inet_hash,
1054f870fa0bSMat Martineau 	.unhash		= inet_unhash,
1055cec37a6eSPeter Krystad 	.get_port	= mptcp_get_port,
10561891c4a0SFlorian Westphal 	.stream_memory_free	= mptcp_memory_free,
1057f870fa0bSMat Martineau 	.obj_size	= sizeof(struct mptcp_sock),
1058f870fa0bSMat Martineau 	.no_autobind	= true,
1059f870fa0bSMat Martineau };
1060f870fa0bSMat Martineau 
10612303f994SPeter Krystad static int mptcp_bind(struct socket *sock, struct sockaddr *uaddr, int addr_len)
10622303f994SPeter Krystad {
10632303f994SPeter Krystad 	struct mptcp_sock *msk = mptcp_sk(sock->sk);
10642303f994SPeter Krystad 	struct socket *ssock;
1065cf7da0d6SPeter Krystad 	int err;
10662303f994SPeter Krystad 
10672303f994SPeter Krystad 	lock_sock(sock->sk);
10682303f994SPeter Krystad 	ssock = __mptcp_socket_create(msk, MPTCP_SAME_STATE);
10692303f994SPeter Krystad 	if (IS_ERR(ssock)) {
10702303f994SPeter Krystad 		err = PTR_ERR(ssock);
10712303f994SPeter Krystad 		goto unlock;
10722303f994SPeter Krystad 	}
10732303f994SPeter Krystad 
10742303f994SPeter Krystad 	err = ssock->ops->bind(ssock, uaddr, addr_len);
1075cf7da0d6SPeter Krystad 	if (!err)
1076cf7da0d6SPeter Krystad 		mptcp_copy_inaddrs(sock->sk, ssock->sk);
10772303f994SPeter Krystad 
10782303f994SPeter Krystad unlock:
10792303f994SPeter Krystad 	release_sock(sock->sk);
10802303f994SPeter Krystad 	return err;
10812303f994SPeter Krystad }
10822303f994SPeter Krystad 
10832303f994SPeter Krystad static int mptcp_stream_connect(struct socket *sock, struct sockaddr *uaddr,
10842303f994SPeter Krystad 				int addr_len, int flags)
10852303f994SPeter Krystad {
10862303f994SPeter Krystad 	struct mptcp_sock *msk = mptcp_sk(sock->sk);
10872303f994SPeter Krystad 	struct socket *ssock;
10882303f994SPeter Krystad 	int err;
10892303f994SPeter Krystad 
10902303f994SPeter Krystad 	lock_sock(sock->sk);
10912303f994SPeter Krystad 	ssock = __mptcp_socket_create(msk, TCP_SYN_SENT);
10922303f994SPeter Krystad 	if (IS_ERR(ssock)) {
10932303f994SPeter Krystad 		err = PTR_ERR(ssock);
10942303f994SPeter Krystad 		goto unlock;
10952303f994SPeter Krystad 	}
10962303f994SPeter Krystad 
1097cf7da0d6SPeter Krystad #ifdef CONFIG_TCP_MD5SIG
1098cf7da0d6SPeter Krystad 	/* no MPTCP if MD5SIG is enabled on this socket or we may run out of
1099cf7da0d6SPeter Krystad 	 * TCP option space.
1100cf7da0d6SPeter Krystad 	 */
1101cf7da0d6SPeter Krystad 	if (rcu_access_pointer(tcp_sk(ssock->sk)->md5sig_info))
1102cf7da0d6SPeter Krystad 		mptcp_subflow_ctx(ssock->sk)->request_mptcp = 0;
1103cf7da0d6SPeter Krystad #endif
1104cf7da0d6SPeter Krystad 
11052303f994SPeter Krystad 	err = ssock->ops->connect(ssock, uaddr, addr_len, flags);
11062303f994SPeter Krystad 	inet_sk_state_store(sock->sk, inet_sk_state_load(ssock->sk));
1107cf7da0d6SPeter Krystad 	mptcp_copy_inaddrs(sock->sk, ssock->sk);
11082303f994SPeter Krystad 
11092303f994SPeter Krystad unlock:
11102303f994SPeter Krystad 	release_sock(sock->sk);
11112303f994SPeter Krystad 	return err;
11122303f994SPeter Krystad }
11132303f994SPeter Krystad 
1114cf7da0d6SPeter Krystad static int mptcp_v4_getname(struct socket *sock, struct sockaddr *uaddr,
1115cf7da0d6SPeter Krystad 			    int peer)
1116cf7da0d6SPeter Krystad {
1117cf7da0d6SPeter Krystad 	if (sock->sk->sk_prot == &tcp_prot) {
1118cf7da0d6SPeter Krystad 		/* we are being invoked from __sys_accept4, after
1119cf7da0d6SPeter Krystad 		 * mptcp_accept() has just accepted a non-mp-capable
1120cf7da0d6SPeter Krystad 		 * flow: sk is a tcp_sk, not an mptcp one.
1121cf7da0d6SPeter Krystad 		 *
1122cf7da0d6SPeter Krystad 		 * Hand the socket over to tcp so all further socket ops
1123cf7da0d6SPeter Krystad 		 * bypass mptcp.
1124cf7da0d6SPeter Krystad 		 */
1125cf7da0d6SPeter Krystad 		sock->ops = &inet_stream_ops;
1126cf7da0d6SPeter Krystad 	}
1127cf7da0d6SPeter Krystad 
1128cf7da0d6SPeter Krystad 	return inet_getname(sock, uaddr, peer);
1129cf7da0d6SPeter Krystad }
1130cf7da0d6SPeter Krystad 
1131cf7da0d6SPeter Krystad #if IS_ENABLED(CONFIG_MPTCP_IPV6)
1132cf7da0d6SPeter Krystad static int mptcp_v6_getname(struct socket *sock, struct sockaddr *uaddr,
1133cf7da0d6SPeter Krystad 			    int peer)
1134cf7da0d6SPeter Krystad {
1135cf7da0d6SPeter Krystad 	if (sock->sk->sk_prot == &tcpv6_prot) {
1136cf7da0d6SPeter Krystad 		/* we are being invoked from __sys_accept4 after
1137cf7da0d6SPeter Krystad 		 * mptcp_accept() has accepted a non-mp-capable
1138cf7da0d6SPeter Krystad 		 * subflow: sk is a tcp_sk, not mptcp.
1139cf7da0d6SPeter Krystad 		 *
1140cf7da0d6SPeter Krystad 		 * Hand the socket over to tcp so all further
1141cf7da0d6SPeter Krystad 		 * socket ops bypass mptcp.
1142cf7da0d6SPeter Krystad 		 */
1143cf7da0d6SPeter Krystad 		sock->ops = &inet6_stream_ops;
1144cf7da0d6SPeter Krystad 	}
1145cf7da0d6SPeter Krystad 
1146cf7da0d6SPeter Krystad 	return inet6_getname(sock, uaddr, peer);
1147cf7da0d6SPeter Krystad }
1148cf7da0d6SPeter Krystad #endif
1149cf7da0d6SPeter Krystad 
1150cf7da0d6SPeter Krystad static int mptcp_listen(struct socket *sock, int backlog)
1151cf7da0d6SPeter Krystad {
1152cf7da0d6SPeter Krystad 	struct mptcp_sock *msk = mptcp_sk(sock->sk);
1153cf7da0d6SPeter Krystad 	struct socket *ssock;
1154cf7da0d6SPeter Krystad 	int err;
1155cf7da0d6SPeter Krystad 
1156cf7da0d6SPeter Krystad 	pr_debug("msk=%p", msk);
1157cf7da0d6SPeter Krystad 
1158cf7da0d6SPeter Krystad 	lock_sock(sock->sk);
1159cf7da0d6SPeter Krystad 	ssock = __mptcp_socket_create(msk, TCP_LISTEN);
1160cf7da0d6SPeter Krystad 	if (IS_ERR(ssock)) {
1161cf7da0d6SPeter Krystad 		err = PTR_ERR(ssock);
1162cf7da0d6SPeter Krystad 		goto unlock;
1163cf7da0d6SPeter Krystad 	}
1164cf7da0d6SPeter Krystad 
1165cf7da0d6SPeter Krystad 	err = ssock->ops->listen(ssock, backlog);
1166cf7da0d6SPeter Krystad 	inet_sk_state_store(sock->sk, inet_sk_state_load(ssock->sk));
1167cf7da0d6SPeter Krystad 	if (!err)
1168cf7da0d6SPeter Krystad 		mptcp_copy_inaddrs(sock->sk, ssock->sk);
1169cf7da0d6SPeter Krystad 
1170cf7da0d6SPeter Krystad unlock:
1171cf7da0d6SPeter Krystad 	release_sock(sock->sk);
1172cf7da0d6SPeter Krystad 	return err;
1173cf7da0d6SPeter Krystad }
1174cf7da0d6SPeter Krystad 
1175cf7da0d6SPeter Krystad static bool is_tcp_proto(const struct proto *p)
1176cf7da0d6SPeter Krystad {
1177cf7da0d6SPeter Krystad #if IS_ENABLED(CONFIG_MPTCP_IPV6)
1178cf7da0d6SPeter Krystad 	return p == &tcp_prot || p == &tcpv6_prot;
1179cf7da0d6SPeter Krystad #else
1180cf7da0d6SPeter Krystad 	return p == &tcp_prot;
1181cf7da0d6SPeter Krystad #endif
1182cf7da0d6SPeter Krystad }
1183cf7da0d6SPeter Krystad 
1184cf7da0d6SPeter Krystad static int mptcp_stream_accept(struct socket *sock, struct socket *newsock,
1185cf7da0d6SPeter Krystad 			       int flags, bool kern)
1186cf7da0d6SPeter Krystad {
1187cf7da0d6SPeter Krystad 	struct mptcp_sock *msk = mptcp_sk(sock->sk);
1188cf7da0d6SPeter Krystad 	struct socket *ssock;
1189cf7da0d6SPeter Krystad 	int err;
1190cf7da0d6SPeter Krystad 
1191cf7da0d6SPeter Krystad 	pr_debug("msk=%p", msk);
1192cf7da0d6SPeter Krystad 
1193cf7da0d6SPeter Krystad 	lock_sock(sock->sk);
1194cf7da0d6SPeter Krystad 	if (sock->sk->sk_state != TCP_LISTEN)
1195cf7da0d6SPeter Krystad 		goto unlock_fail;
1196cf7da0d6SPeter Krystad 
1197cf7da0d6SPeter Krystad 	ssock = __mptcp_nmpc_socket(msk);
1198cf7da0d6SPeter Krystad 	if (!ssock)
1199cf7da0d6SPeter Krystad 		goto unlock_fail;
1200cf7da0d6SPeter Krystad 
1201cf7da0d6SPeter Krystad 	sock_hold(ssock->sk);
1202cf7da0d6SPeter Krystad 	release_sock(sock->sk);
1203cf7da0d6SPeter Krystad 
1204cf7da0d6SPeter Krystad 	err = ssock->ops->accept(sock, newsock, flags, kern);
1205cf7da0d6SPeter Krystad 	if (err == 0 && !is_tcp_proto(newsock->sk->sk_prot)) {
1206cf7da0d6SPeter Krystad 		struct mptcp_sock *msk = mptcp_sk(newsock->sk);
1207cf7da0d6SPeter Krystad 		struct mptcp_subflow_context *subflow;
1208cf7da0d6SPeter Krystad 
1209cf7da0d6SPeter Krystad 		/* set ssk->sk_socket of accept()ed flows to mptcp socket.
1210cf7da0d6SPeter Krystad 		 * This is needed so NOSPACE flag can be set from tcp stack.
1211cf7da0d6SPeter Krystad 		 */
1212cf7da0d6SPeter Krystad 		list_for_each_entry(subflow, &msk->conn_list, node) {
1213cf7da0d6SPeter Krystad 			struct sock *ssk = mptcp_subflow_tcp_sock(subflow);
1214cf7da0d6SPeter Krystad 
1215cf7da0d6SPeter Krystad 			if (!ssk->sk_socket)
1216cf7da0d6SPeter Krystad 				mptcp_sock_graft(ssk, newsock);
1217cf7da0d6SPeter Krystad 		}
1218cf7da0d6SPeter Krystad 
1219cf7da0d6SPeter Krystad 		inet_sk_state_store(newsock->sk, TCP_ESTABLISHED);
1220cf7da0d6SPeter Krystad 	}
1221cf7da0d6SPeter Krystad 
1222cf7da0d6SPeter Krystad 	sock_put(ssock->sk);
1223cf7da0d6SPeter Krystad 	return err;
1224cf7da0d6SPeter Krystad 
1225cf7da0d6SPeter Krystad unlock_fail:
1226cf7da0d6SPeter Krystad 	release_sock(sock->sk);
1227cf7da0d6SPeter Krystad 	return -EINVAL;
1228cf7da0d6SPeter Krystad }
1229cf7da0d6SPeter Krystad 
12302303f994SPeter Krystad static __poll_t mptcp_poll(struct file *file, struct socket *sock,
12312303f994SPeter Krystad 			   struct poll_table_struct *wait)
12322303f994SPeter Krystad {
12331891c4a0SFlorian Westphal 	struct sock *sk = sock->sk;
12348ab183deSPaolo Abeni 	struct mptcp_sock *msk;
12351891c4a0SFlorian Westphal 	struct socket *ssock;
12362303f994SPeter Krystad 	__poll_t mask = 0;
12372303f994SPeter Krystad 
12381891c4a0SFlorian Westphal 	msk = mptcp_sk(sk);
12391891c4a0SFlorian Westphal 	lock_sock(sk);
12401891c4a0SFlorian Westphal 	ssock = __mptcp_nmpc_socket(msk);
12411891c4a0SFlorian Westphal 	if (ssock) {
12421891c4a0SFlorian Westphal 		mask = ssock->ops->poll(file, ssock, wait);
12431891c4a0SFlorian Westphal 		release_sock(sk);
12441891c4a0SFlorian Westphal 		return mask;
12451891c4a0SFlorian Westphal 	}
12461891c4a0SFlorian Westphal 
12471891c4a0SFlorian Westphal 	release_sock(sk);
12481891c4a0SFlorian Westphal 	sock_poll_wait(file, sock, wait);
12491891c4a0SFlorian Westphal 	lock_sock(sk);
12508ab183deSPaolo Abeni 	ssock = __mptcp_tcp_fallback(msk);
12518ab183deSPaolo Abeni 	if (unlikely(ssock))
12528ab183deSPaolo Abeni 		return ssock->ops->poll(file, ssock, NULL);
12531891c4a0SFlorian Westphal 
12541891c4a0SFlorian Westphal 	if (test_bit(MPTCP_DATA_READY, &msk->flags))
12551891c4a0SFlorian Westphal 		mask = EPOLLIN | EPOLLRDNORM;
12561891c4a0SFlorian Westphal 	if (sk_stream_is_writeable(sk) &&
12571891c4a0SFlorian Westphal 	    test_bit(MPTCP_SEND_SPACE, &msk->flags))
12581891c4a0SFlorian Westphal 		mask |= EPOLLOUT | EPOLLWRNORM;
12591891c4a0SFlorian Westphal 	if (sk->sk_shutdown & RCV_SHUTDOWN)
12601891c4a0SFlorian Westphal 		mask |= EPOLLIN | EPOLLRDNORM | EPOLLRDHUP;
12611891c4a0SFlorian Westphal 
12621891c4a0SFlorian Westphal 	release_sock(sk);
12631891c4a0SFlorian Westphal 
12642303f994SPeter Krystad 	return mask;
12652303f994SPeter Krystad }
12662303f994SPeter Krystad 
126721498490SPeter Krystad static int mptcp_shutdown(struct socket *sock, int how)
126821498490SPeter Krystad {
126921498490SPeter Krystad 	struct mptcp_sock *msk = mptcp_sk(sock->sk);
127021498490SPeter Krystad 	struct mptcp_subflow_context *subflow;
127121498490SPeter Krystad 	int ret = 0;
127221498490SPeter Krystad 
127321498490SPeter Krystad 	pr_debug("sk=%p, how=%d", msk, how);
127421498490SPeter Krystad 
127521498490SPeter Krystad 	lock_sock(sock->sk);
127621498490SPeter Krystad 
127721498490SPeter Krystad 	if (how == SHUT_WR || how == SHUT_RDWR)
127821498490SPeter Krystad 		inet_sk_state_store(sock->sk, TCP_FIN_WAIT1);
127921498490SPeter Krystad 
128021498490SPeter Krystad 	how++;
128121498490SPeter Krystad 
128221498490SPeter Krystad 	if ((how & ~SHUTDOWN_MASK) || !how) {
128321498490SPeter Krystad 		ret = -EINVAL;
128421498490SPeter Krystad 		goto out_unlock;
128521498490SPeter Krystad 	}
128621498490SPeter Krystad 
128721498490SPeter Krystad 	if (sock->state == SS_CONNECTING) {
128821498490SPeter Krystad 		if ((1 << sock->sk->sk_state) &
128921498490SPeter Krystad 		    (TCPF_SYN_SENT | TCPF_SYN_RECV | TCPF_CLOSE))
129021498490SPeter Krystad 			sock->state = SS_DISCONNECTING;
129121498490SPeter Krystad 		else
129221498490SPeter Krystad 			sock->state = SS_CONNECTED;
129321498490SPeter Krystad 	}
129421498490SPeter Krystad 
129521498490SPeter Krystad 	mptcp_for_each_subflow(msk, subflow) {
129621498490SPeter Krystad 		struct sock *tcp_sk = mptcp_subflow_tcp_sock(subflow);
129721498490SPeter Krystad 
129821498490SPeter Krystad 		mptcp_subflow_shutdown(tcp_sk, how);
129921498490SPeter Krystad 	}
130021498490SPeter Krystad 
130121498490SPeter Krystad out_unlock:
130221498490SPeter Krystad 	release_sock(sock->sk);
130321498490SPeter Krystad 
130421498490SPeter Krystad 	return ret;
130521498490SPeter Krystad }
130621498490SPeter Krystad 
1307e42f1ac6SFlorian Westphal static const struct proto_ops mptcp_stream_ops = {
1308e42f1ac6SFlorian Westphal 	.family		   = PF_INET,
1309e42f1ac6SFlorian Westphal 	.owner		   = THIS_MODULE,
1310e42f1ac6SFlorian Westphal 	.release	   = inet_release,
1311e42f1ac6SFlorian Westphal 	.bind		   = mptcp_bind,
1312e42f1ac6SFlorian Westphal 	.connect	   = mptcp_stream_connect,
1313e42f1ac6SFlorian Westphal 	.socketpair	   = sock_no_socketpair,
1314e42f1ac6SFlorian Westphal 	.accept		   = mptcp_stream_accept,
1315e42f1ac6SFlorian Westphal 	.getname	   = mptcp_v4_getname,
1316e42f1ac6SFlorian Westphal 	.poll		   = mptcp_poll,
1317e42f1ac6SFlorian Westphal 	.ioctl		   = inet_ioctl,
1318e42f1ac6SFlorian Westphal 	.gettstamp	   = sock_gettstamp,
1319e42f1ac6SFlorian Westphal 	.listen		   = mptcp_listen,
1320e42f1ac6SFlorian Westphal 	.shutdown	   = mptcp_shutdown,
1321e42f1ac6SFlorian Westphal 	.setsockopt	   = sock_common_setsockopt,
1322e42f1ac6SFlorian Westphal 	.getsockopt	   = sock_common_getsockopt,
1323e42f1ac6SFlorian Westphal 	.sendmsg	   = inet_sendmsg,
1324e42f1ac6SFlorian Westphal 	.recvmsg	   = inet_recvmsg,
1325e42f1ac6SFlorian Westphal 	.mmap		   = sock_no_mmap,
1326e42f1ac6SFlorian Westphal 	.sendpage	   = inet_sendpage,
1327e42f1ac6SFlorian Westphal #ifdef CONFIG_COMPAT
1328e42f1ac6SFlorian Westphal 	.compat_setsockopt = compat_sock_common_setsockopt,
1329e42f1ac6SFlorian Westphal 	.compat_getsockopt = compat_sock_common_getsockopt,
1330e42f1ac6SFlorian Westphal #endif
1331e42f1ac6SFlorian Westphal };
13322303f994SPeter Krystad 
1333f870fa0bSMat Martineau static struct inet_protosw mptcp_protosw = {
1334f870fa0bSMat Martineau 	.type		= SOCK_STREAM,
1335f870fa0bSMat Martineau 	.protocol	= IPPROTO_MPTCP,
1336f870fa0bSMat Martineau 	.prot		= &mptcp_prot,
13372303f994SPeter Krystad 	.ops		= &mptcp_stream_ops,
13382303f994SPeter Krystad 	.flags		= INET_PROTOSW_ICSK,
1339f870fa0bSMat Martineau };
1340f870fa0bSMat Martineau 
1341784325e9SMatthieu Baerts void mptcp_proto_init(void)
1342f870fa0bSMat Martineau {
13432303f994SPeter Krystad 	mptcp_prot.h.hashinfo = tcp_prot.h.hashinfo;
13442303f994SPeter Krystad 
13452303f994SPeter Krystad 	mptcp_subflow_init();
13462303f994SPeter Krystad 
1347f870fa0bSMat Martineau 	if (proto_register(&mptcp_prot, 1) != 0)
1348f870fa0bSMat Martineau 		panic("Failed to register MPTCP proto.\n");
1349f870fa0bSMat Martineau 
1350f870fa0bSMat Martineau 	inet_register_protosw(&mptcp_protosw);
13516771bfd9SFlorian Westphal 
13526771bfd9SFlorian Westphal 	BUILD_BUG_ON(sizeof(struct mptcp_skb_cb) > sizeof_field(struct sk_buff, cb));
1353f870fa0bSMat Martineau }
1354f870fa0bSMat Martineau 
1355f870fa0bSMat Martineau #if IS_ENABLED(CONFIG_MPTCP_IPV6)
1356e42f1ac6SFlorian Westphal static const struct proto_ops mptcp_v6_stream_ops = {
1357e42f1ac6SFlorian Westphal 	.family		   = PF_INET6,
1358e42f1ac6SFlorian Westphal 	.owner		   = THIS_MODULE,
1359e42f1ac6SFlorian Westphal 	.release	   = inet6_release,
1360e42f1ac6SFlorian Westphal 	.bind		   = mptcp_bind,
1361e42f1ac6SFlorian Westphal 	.connect	   = mptcp_stream_connect,
1362e42f1ac6SFlorian Westphal 	.socketpair	   = sock_no_socketpair,
1363e42f1ac6SFlorian Westphal 	.accept		   = mptcp_stream_accept,
1364e42f1ac6SFlorian Westphal 	.getname	   = mptcp_v6_getname,
1365e42f1ac6SFlorian Westphal 	.poll		   = mptcp_poll,
1366e42f1ac6SFlorian Westphal 	.ioctl		   = inet6_ioctl,
1367e42f1ac6SFlorian Westphal 	.gettstamp	   = sock_gettstamp,
1368e42f1ac6SFlorian Westphal 	.listen		   = mptcp_listen,
1369e42f1ac6SFlorian Westphal 	.shutdown	   = mptcp_shutdown,
1370e42f1ac6SFlorian Westphal 	.setsockopt	   = sock_common_setsockopt,
1371e42f1ac6SFlorian Westphal 	.getsockopt	   = sock_common_getsockopt,
1372e42f1ac6SFlorian Westphal 	.sendmsg	   = inet6_sendmsg,
1373e42f1ac6SFlorian Westphal 	.recvmsg	   = inet6_recvmsg,
1374e42f1ac6SFlorian Westphal 	.mmap		   = sock_no_mmap,
1375e42f1ac6SFlorian Westphal 	.sendpage	   = inet_sendpage,
1376e42f1ac6SFlorian Westphal #ifdef CONFIG_COMPAT
1377e42f1ac6SFlorian Westphal 	.compat_setsockopt = compat_sock_common_setsockopt,
1378e42f1ac6SFlorian Westphal 	.compat_getsockopt = compat_sock_common_getsockopt,
1379e42f1ac6SFlorian Westphal #endif
1380e42f1ac6SFlorian Westphal };
1381e42f1ac6SFlorian Westphal 
1382f870fa0bSMat Martineau static struct proto mptcp_v6_prot;
1383f870fa0bSMat Martineau 
138479c0949eSPeter Krystad static void mptcp_v6_destroy(struct sock *sk)
138579c0949eSPeter Krystad {
138679c0949eSPeter Krystad 	mptcp_destroy(sk);
138779c0949eSPeter Krystad 	inet6_destroy_sock(sk);
138879c0949eSPeter Krystad }
138979c0949eSPeter Krystad 
1390f870fa0bSMat Martineau static struct inet_protosw mptcp_v6_protosw = {
1391f870fa0bSMat Martineau 	.type		= SOCK_STREAM,
1392f870fa0bSMat Martineau 	.protocol	= IPPROTO_MPTCP,
1393f870fa0bSMat Martineau 	.prot		= &mptcp_v6_prot,
13942303f994SPeter Krystad 	.ops		= &mptcp_v6_stream_ops,
1395f870fa0bSMat Martineau 	.flags		= INET_PROTOSW_ICSK,
1396f870fa0bSMat Martineau };
1397f870fa0bSMat Martineau 
1398784325e9SMatthieu Baerts int mptcp_proto_v6_init(void)
1399f870fa0bSMat Martineau {
1400f870fa0bSMat Martineau 	int err;
1401f870fa0bSMat Martineau 
1402f870fa0bSMat Martineau 	mptcp_v6_prot = mptcp_prot;
1403f870fa0bSMat Martineau 	strcpy(mptcp_v6_prot.name, "MPTCPv6");
1404f870fa0bSMat Martineau 	mptcp_v6_prot.slab = NULL;
140579c0949eSPeter Krystad 	mptcp_v6_prot.destroy = mptcp_v6_destroy;
1406b0519de8SFlorian Westphal 	mptcp_v6_prot.obj_size = sizeof(struct mptcp6_sock);
1407f870fa0bSMat Martineau 
1408f870fa0bSMat Martineau 	err = proto_register(&mptcp_v6_prot, 1);
1409f870fa0bSMat Martineau 	if (err)
1410f870fa0bSMat Martineau 		return err;
1411f870fa0bSMat Martineau 
1412f870fa0bSMat Martineau 	err = inet6_register_protosw(&mptcp_v6_protosw);
1413f870fa0bSMat Martineau 	if (err)
1414f870fa0bSMat Martineau 		proto_unregister(&mptcp_v6_prot);
1415f870fa0bSMat Martineau 
1416f870fa0bSMat Martineau 	return err;
1417f870fa0bSMat Martineau }
1418f870fa0bSMat Martineau #endif
1419