1f870fa0bSMat Martineau // SPDX-License-Identifier: GPL-2.0 2f870fa0bSMat Martineau /* Multipath TCP 3f870fa0bSMat Martineau * 4f870fa0bSMat Martineau * Copyright (c) 2017 - 2019, Intel Corporation. 5f870fa0bSMat Martineau */ 6f870fa0bSMat Martineau 7f870fa0bSMat Martineau #define pr_fmt(fmt) "MPTCP: " fmt 8f870fa0bSMat Martineau 9f870fa0bSMat Martineau #include <linux/kernel.h> 10f870fa0bSMat Martineau #include <linux/module.h> 11f870fa0bSMat Martineau #include <linux/netdevice.h> 127a6a6cbcSPaolo Abeni #include <linux/sched/signal.h> 137a6a6cbcSPaolo Abeni #include <linux/atomic.h> 14f870fa0bSMat Martineau #include <net/sock.h> 15f870fa0bSMat Martineau #include <net/inet_common.h> 16f870fa0bSMat Martineau #include <net/inet_hashtables.h> 17f870fa0bSMat Martineau #include <net/protocol.h> 18f870fa0bSMat Martineau #include <net/tcp.h> 19cf7da0d6SPeter Krystad #if IS_ENABLED(CONFIG_MPTCP_IPV6) 20cf7da0d6SPeter Krystad #include <net/transp_v6.h> 21cf7da0d6SPeter Krystad #endif 22f870fa0bSMat Martineau #include <net/mptcp.h> 23f870fa0bSMat Martineau #include "protocol.h" 24f870fa0bSMat Martineau 252303f994SPeter Krystad #define MPTCP_SAME_STATE TCP_MAX_STATES 262303f994SPeter Krystad 27b0519de8SFlorian Westphal #if IS_ENABLED(CONFIG_MPTCP_IPV6) 28b0519de8SFlorian Westphal struct mptcp6_sock { 29b0519de8SFlorian Westphal struct mptcp_sock msk; 30b0519de8SFlorian Westphal struct ipv6_pinfo np; 31b0519de8SFlorian Westphal }; 32b0519de8SFlorian Westphal #endif 33b0519de8SFlorian Westphal 346771bfd9SFlorian Westphal struct mptcp_skb_cb { 356771bfd9SFlorian Westphal u32 offset; 366771bfd9SFlorian Westphal }; 376771bfd9SFlorian Westphal 386771bfd9SFlorian Westphal #define MPTCP_SKB_CB(__skb) ((struct mptcp_skb_cb *)&((__skb)->cb[0])) 396771bfd9SFlorian Westphal 402303f994SPeter Krystad /* If msk has an initial subflow socket, and the MP_CAPABLE handshake has not 412303f994SPeter Krystad * completed yet or has failed, return the subflow socket. 422303f994SPeter Krystad * Otherwise return NULL. 432303f994SPeter Krystad */ 442303f994SPeter Krystad static struct socket *__mptcp_nmpc_socket(const struct mptcp_sock *msk) 452303f994SPeter Krystad { 46d22f4988SChristoph Paasch if (!msk->subflow || READ_ONCE(msk->can_ack)) 472303f994SPeter Krystad return NULL; 482303f994SPeter Krystad 492303f994SPeter Krystad return msk->subflow; 502303f994SPeter Krystad } 512303f994SPeter Krystad 528ab183deSPaolo Abeni static bool __mptcp_needs_tcp_fallback(const struct mptcp_sock *msk) 538ab183deSPaolo Abeni { 548ab183deSPaolo Abeni return msk->first && !sk_is_mptcp(msk->first); 558ab183deSPaolo Abeni } 568ab183deSPaolo Abeni 578ab183deSPaolo Abeni static struct socket *__mptcp_tcp_fallback(struct mptcp_sock *msk) 58cec37a6eSPeter Krystad { 59cec37a6eSPeter Krystad sock_owned_by_me((const struct sock *)msk); 60cec37a6eSPeter Krystad 618ab183deSPaolo Abeni if (likely(!__mptcp_needs_tcp_fallback(msk))) 62cec37a6eSPeter Krystad return NULL; 63cec37a6eSPeter Krystad 648ab183deSPaolo Abeni if (msk->subflow) { 652c22c06cSFlorian Westphal release_sock((struct sock *)msk); 662c22c06cSFlorian Westphal return msk->subflow; 678ab183deSPaolo Abeni } 688ab183deSPaolo Abeni 692c22c06cSFlorian Westphal return NULL; 70cec37a6eSPeter Krystad } 71cec37a6eSPeter Krystad 722303f994SPeter Krystad static bool __mptcp_can_create_subflow(const struct mptcp_sock *msk) 732303f994SPeter Krystad { 748ab183deSPaolo Abeni return !msk->first; 752303f994SPeter Krystad } 762303f994SPeter Krystad 772303f994SPeter Krystad static struct socket *__mptcp_socket_create(struct mptcp_sock *msk, int state) 782303f994SPeter Krystad { 792303f994SPeter Krystad struct mptcp_subflow_context *subflow; 802303f994SPeter Krystad struct sock *sk = (struct sock *)msk; 812303f994SPeter Krystad struct socket *ssock; 822303f994SPeter Krystad int err; 832303f994SPeter Krystad 842303f994SPeter Krystad ssock = __mptcp_nmpc_socket(msk); 852303f994SPeter Krystad if (ssock) 862303f994SPeter Krystad goto set_state; 872303f994SPeter Krystad 882303f994SPeter Krystad if (!__mptcp_can_create_subflow(msk)) 892303f994SPeter Krystad return ERR_PTR(-EINVAL); 902303f994SPeter Krystad 912303f994SPeter Krystad err = mptcp_subflow_create_socket(sk, &ssock); 922303f994SPeter Krystad if (err) 932303f994SPeter Krystad return ERR_PTR(err); 942303f994SPeter Krystad 958ab183deSPaolo Abeni msk->first = ssock->sk; 962303f994SPeter Krystad msk->subflow = ssock; 972303f994SPeter Krystad subflow = mptcp_subflow_ctx(ssock->sk); 98cec37a6eSPeter Krystad list_add(&subflow->node, &msk->conn_list); 992303f994SPeter Krystad subflow->request_mptcp = 1; 1002303f994SPeter Krystad 1012303f994SPeter Krystad set_state: 1022303f994SPeter Krystad if (state != MPTCP_SAME_STATE) 1032303f994SPeter Krystad inet_sk_state_store(sk, state); 1042303f994SPeter Krystad return ssock; 1052303f994SPeter Krystad } 1062303f994SPeter Krystad 1076771bfd9SFlorian Westphal static void __mptcp_move_skb(struct mptcp_sock *msk, struct sock *ssk, 1086771bfd9SFlorian Westphal struct sk_buff *skb, 1096771bfd9SFlorian Westphal unsigned int offset, size_t copy_len) 1106771bfd9SFlorian Westphal { 1116771bfd9SFlorian Westphal struct sock *sk = (struct sock *)msk; 1126771bfd9SFlorian Westphal 1136771bfd9SFlorian Westphal __skb_unlink(skb, &ssk->sk_receive_queue); 114600911ffSFlorian Westphal skb_set_owner_r(skb, sk); 1156771bfd9SFlorian Westphal __skb_queue_tail(&sk->sk_receive_queue, skb); 1166771bfd9SFlorian Westphal 1176771bfd9SFlorian Westphal msk->ack_seq += copy_len; 1186771bfd9SFlorian Westphal MPTCP_SKB_CB(skb)->offset = offset; 1196771bfd9SFlorian Westphal } 1206771bfd9SFlorian Westphal 1216771bfd9SFlorian Westphal static bool __mptcp_move_skbs_from_subflow(struct mptcp_sock *msk, 1226771bfd9SFlorian Westphal struct sock *ssk, 1236771bfd9SFlorian Westphal unsigned int *bytes) 1246771bfd9SFlorian Westphal { 1256771bfd9SFlorian Westphal struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk); 126600911ffSFlorian Westphal struct sock *sk = (struct sock *)msk; 1276771bfd9SFlorian Westphal unsigned int moved = 0; 1286771bfd9SFlorian Westphal bool more_data_avail; 1296771bfd9SFlorian Westphal struct tcp_sock *tp; 1306771bfd9SFlorian Westphal bool done = false; 131600911ffSFlorian Westphal 132ec33916dSFlorian Westphal if (!(sk->sk_userlocks & SOCK_RCVBUF_LOCK)) { 133ec33916dSFlorian Westphal int rcvbuf = max(ssk->sk_rcvbuf, sk->sk_rcvbuf); 134ec33916dSFlorian Westphal 135600911ffSFlorian Westphal if (rcvbuf > sk->sk_rcvbuf) 136600911ffSFlorian Westphal sk->sk_rcvbuf = rcvbuf; 137ec33916dSFlorian Westphal } 1386771bfd9SFlorian Westphal 1396771bfd9SFlorian Westphal tp = tcp_sk(ssk); 1406771bfd9SFlorian Westphal do { 1416771bfd9SFlorian Westphal u32 map_remaining, offset; 1426771bfd9SFlorian Westphal u32 seq = tp->copied_seq; 1436771bfd9SFlorian Westphal struct sk_buff *skb; 1446771bfd9SFlorian Westphal bool fin; 1456771bfd9SFlorian Westphal 1466771bfd9SFlorian Westphal /* try to move as much data as available */ 1476771bfd9SFlorian Westphal map_remaining = subflow->map_data_len - 1486771bfd9SFlorian Westphal mptcp_subflow_get_map_offset(subflow); 1496771bfd9SFlorian Westphal 1506771bfd9SFlorian Westphal skb = skb_peek(&ssk->sk_receive_queue); 1516771bfd9SFlorian Westphal if (!skb) 1526771bfd9SFlorian Westphal break; 1536771bfd9SFlorian Westphal 1546771bfd9SFlorian Westphal offset = seq - TCP_SKB_CB(skb)->seq; 1556771bfd9SFlorian Westphal fin = TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN; 1566771bfd9SFlorian Westphal if (fin) { 1576771bfd9SFlorian Westphal done = true; 1586771bfd9SFlorian Westphal seq++; 1596771bfd9SFlorian Westphal } 1606771bfd9SFlorian Westphal 1616771bfd9SFlorian Westphal if (offset < skb->len) { 1626771bfd9SFlorian Westphal size_t len = skb->len - offset; 1636771bfd9SFlorian Westphal 1646771bfd9SFlorian Westphal if (tp->urg_data) 1656771bfd9SFlorian Westphal done = true; 1666771bfd9SFlorian Westphal 1676771bfd9SFlorian Westphal __mptcp_move_skb(msk, ssk, skb, offset, len); 1686771bfd9SFlorian Westphal seq += len; 1696771bfd9SFlorian Westphal moved += len; 1706771bfd9SFlorian Westphal 1716771bfd9SFlorian Westphal if (WARN_ON_ONCE(map_remaining < len)) 1726771bfd9SFlorian Westphal break; 1736771bfd9SFlorian Westphal } else { 1746771bfd9SFlorian Westphal WARN_ON_ONCE(!fin); 1756771bfd9SFlorian Westphal sk_eat_skb(ssk, skb); 1766771bfd9SFlorian Westphal done = true; 1776771bfd9SFlorian Westphal } 1786771bfd9SFlorian Westphal 1796771bfd9SFlorian Westphal WRITE_ONCE(tp->copied_seq, seq); 1806771bfd9SFlorian Westphal more_data_avail = mptcp_subflow_data_available(ssk); 181600911ffSFlorian Westphal 182600911ffSFlorian Westphal if (atomic_read(&sk->sk_rmem_alloc) > READ_ONCE(sk->sk_rcvbuf)) { 183600911ffSFlorian Westphal done = true; 184600911ffSFlorian Westphal break; 185600911ffSFlorian Westphal } 1866771bfd9SFlorian Westphal } while (more_data_avail); 1876771bfd9SFlorian Westphal 1886771bfd9SFlorian Westphal *bytes = moved; 1896771bfd9SFlorian Westphal 1906771bfd9SFlorian Westphal return done; 1916771bfd9SFlorian Westphal } 1926771bfd9SFlorian Westphal 1932e52213cSFlorian Westphal /* In most cases we will be able to lock the mptcp socket. If its already 1942e52213cSFlorian Westphal * owned, we need to defer to the work queue to avoid ABBA deadlock. 1952e52213cSFlorian Westphal */ 1962e52213cSFlorian Westphal static bool move_skbs_to_msk(struct mptcp_sock *msk, struct sock *ssk) 1972e52213cSFlorian Westphal { 1982e52213cSFlorian Westphal struct sock *sk = (struct sock *)msk; 1992e52213cSFlorian Westphal unsigned int moved = 0; 2002e52213cSFlorian Westphal 2012e52213cSFlorian Westphal if (READ_ONCE(sk->sk_lock.owned)) 2022e52213cSFlorian Westphal return false; 2032e52213cSFlorian Westphal 2042e52213cSFlorian Westphal if (unlikely(!spin_trylock_bh(&sk->sk_lock.slock))) 2052e52213cSFlorian Westphal return false; 2062e52213cSFlorian Westphal 2072e52213cSFlorian Westphal /* must re-check after taking the lock */ 2082e52213cSFlorian Westphal if (!READ_ONCE(sk->sk_lock.owned)) 2092e52213cSFlorian Westphal __mptcp_move_skbs_from_subflow(msk, ssk, &moved); 2102e52213cSFlorian Westphal 2112e52213cSFlorian Westphal spin_unlock_bh(&sk->sk_lock.slock); 2122e52213cSFlorian Westphal 2132e52213cSFlorian Westphal return moved > 0; 2142e52213cSFlorian Westphal } 2152e52213cSFlorian Westphal 2162e52213cSFlorian Westphal void mptcp_data_ready(struct sock *sk, struct sock *ssk) 217101f6f85SFlorian Westphal { 218101f6f85SFlorian Westphal struct mptcp_sock *msk = mptcp_sk(sk); 219101f6f85SFlorian Westphal 220101f6f85SFlorian Westphal set_bit(MPTCP_DATA_READY, &msk->flags); 2216771bfd9SFlorian Westphal 2222e52213cSFlorian Westphal if (atomic_read(&sk->sk_rmem_alloc) < READ_ONCE(sk->sk_rcvbuf) && 2232e52213cSFlorian Westphal move_skbs_to_msk(msk, ssk)) 2242e52213cSFlorian Westphal goto wake; 2252e52213cSFlorian Westphal 226600911ffSFlorian Westphal /* don't schedule if mptcp sk is (still) over limit */ 227600911ffSFlorian Westphal if (atomic_read(&sk->sk_rmem_alloc) > READ_ONCE(sk->sk_rcvbuf)) 228600911ffSFlorian Westphal goto wake; 229600911ffSFlorian Westphal 23014c441b5SPaolo Abeni /* mptcp socket is owned, release_cb should retry */ 23114c441b5SPaolo Abeni if (!test_and_set_bit(TCP_DELACK_TIMER_DEFERRED, 23214c441b5SPaolo Abeni &sk->sk_tsq_flags)) { 23314c441b5SPaolo Abeni sock_hold(sk); 2346771bfd9SFlorian Westphal 23514c441b5SPaolo Abeni /* need to try again, its possible release_cb() has already 23614c441b5SPaolo Abeni * been called after the test_and_set_bit() above. 23714c441b5SPaolo Abeni */ 23814c441b5SPaolo Abeni move_skbs_to_msk(msk, ssk); 23914c441b5SPaolo Abeni } 240600911ffSFlorian Westphal wake: 241101f6f85SFlorian Westphal sk->sk_data_ready(sk); 242101f6f85SFlorian Westphal } 243101f6f85SFlorian Westphal 2446d0060f6SMat Martineau static bool mptcp_ext_cache_refill(struct mptcp_sock *msk) 2456d0060f6SMat Martineau { 2466d0060f6SMat Martineau if (!msk->cached_ext) 2476d0060f6SMat Martineau msk->cached_ext = __skb_ext_alloc(); 2486d0060f6SMat Martineau 2496d0060f6SMat Martineau return !!msk->cached_ext; 2506d0060f6SMat Martineau } 2516d0060f6SMat Martineau 2527a6a6cbcSPaolo Abeni static struct sock *mptcp_subflow_recv_lookup(const struct mptcp_sock *msk) 2537a6a6cbcSPaolo Abeni { 2547a6a6cbcSPaolo Abeni struct mptcp_subflow_context *subflow; 2557a6a6cbcSPaolo Abeni struct sock *sk = (struct sock *)msk; 2567a6a6cbcSPaolo Abeni 2577a6a6cbcSPaolo Abeni sock_owned_by_me(sk); 2587a6a6cbcSPaolo Abeni 2597a6a6cbcSPaolo Abeni mptcp_for_each_subflow(msk, subflow) { 2607a6a6cbcSPaolo Abeni if (subflow->data_avail) 2617a6a6cbcSPaolo Abeni return mptcp_subflow_tcp_sock(subflow); 2627a6a6cbcSPaolo Abeni } 2637a6a6cbcSPaolo Abeni 2647a6a6cbcSPaolo Abeni return NULL; 2657a6a6cbcSPaolo Abeni } 2667a6a6cbcSPaolo Abeni 26757040755SPaolo Abeni static inline bool mptcp_skb_can_collapse_to(const struct mptcp_sock *msk, 26857040755SPaolo Abeni const struct sk_buff *skb, 26957040755SPaolo Abeni const struct mptcp_ext *mpext) 2706d0060f6SMat Martineau { 27157040755SPaolo Abeni if (!tcp_skb_can_collapse_to(skb)) 27257040755SPaolo Abeni return false; 27357040755SPaolo Abeni 27457040755SPaolo Abeni /* can collapse only if MPTCP level sequence is in order */ 27557040755SPaolo Abeni return mpext && mpext->data_seq + mpext->data_len == msk->write_seq; 27657040755SPaolo Abeni } 27757040755SPaolo Abeni 27857040755SPaolo Abeni static int mptcp_sendmsg_frag(struct sock *sk, struct sock *ssk, 27957040755SPaolo Abeni struct msghdr *msg, long *timeo, int *pmss_now, 28057040755SPaolo Abeni int *ps_goal) 28157040755SPaolo Abeni { 28257040755SPaolo Abeni int mss_now, avail_size, size_goal, ret; 2836d0060f6SMat Martineau struct mptcp_sock *msk = mptcp_sk(sk); 2846d0060f6SMat Martineau struct mptcp_ext *mpext = NULL; 28557040755SPaolo Abeni struct sk_buff *skb, *tail; 28657040755SPaolo Abeni bool can_collapse = false; 2876d0060f6SMat Martineau struct page_frag *pfrag; 2886d0060f6SMat Martineau size_t psize; 2896d0060f6SMat Martineau 2906d0060f6SMat Martineau /* use the mptcp page cache so that we can easily move the data 2916d0060f6SMat Martineau * from one substream to another, but do per subflow memory accounting 2926d0060f6SMat Martineau */ 2936d0060f6SMat Martineau pfrag = sk_page_frag(sk); 2946d0060f6SMat Martineau while (!sk_page_frag_refill(ssk, pfrag) || 2956d0060f6SMat Martineau !mptcp_ext_cache_refill(msk)) { 2966d0060f6SMat Martineau ret = sk_stream_wait_memory(ssk, timeo); 2976d0060f6SMat Martineau if (ret) 2986d0060f6SMat Martineau return ret; 2998ab183deSPaolo Abeni if (unlikely(__mptcp_needs_tcp_fallback(msk))) 3008ab183deSPaolo Abeni return 0; 3016d0060f6SMat Martineau } 3026d0060f6SMat Martineau 3036d0060f6SMat Martineau /* compute copy limit */ 3046d0060f6SMat Martineau mss_now = tcp_send_mss(ssk, &size_goal, msg->msg_flags); 30557040755SPaolo Abeni *pmss_now = mss_now; 30657040755SPaolo Abeni *ps_goal = size_goal; 30757040755SPaolo Abeni avail_size = size_goal; 30857040755SPaolo Abeni skb = tcp_write_queue_tail(ssk); 30957040755SPaolo Abeni if (skb) { 31057040755SPaolo Abeni mpext = skb_ext_find(skb, SKB_EXT_MPTCP); 3116d0060f6SMat Martineau 31257040755SPaolo Abeni /* Limit the write to the size available in the 31357040755SPaolo Abeni * current skb, if any, so that we create at most a new skb. 31457040755SPaolo Abeni * Explicitly tells TCP internals to avoid collapsing on later 31557040755SPaolo Abeni * queue management operation, to avoid breaking the ext <-> 31657040755SPaolo Abeni * SSN association set here 31757040755SPaolo Abeni */ 31857040755SPaolo Abeni can_collapse = (size_goal - skb->len > 0) && 31957040755SPaolo Abeni mptcp_skb_can_collapse_to(msk, skb, mpext); 32057040755SPaolo Abeni if (!can_collapse) 32157040755SPaolo Abeni TCP_SKB_CB(skb)->eor = 1; 32257040755SPaolo Abeni else 32357040755SPaolo Abeni avail_size = size_goal - skb->len; 32457040755SPaolo Abeni } 32557040755SPaolo Abeni psize = min_t(size_t, pfrag->size - pfrag->offset, avail_size); 32657040755SPaolo Abeni 32757040755SPaolo Abeni /* Copy to page */ 3286d0060f6SMat Martineau pr_debug("left=%zu", msg_data_left(msg)); 3296d0060f6SMat Martineau psize = copy_page_from_iter(pfrag->page, pfrag->offset, 3306d0060f6SMat Martineau min_t(size_t, msg_data_left(msg), psize), 3316d0060f6SMat Martineau &msg->msg_iter); 3326d0060f6SMat Martineau pr_debug("left=%zu", msg_data_left(msg)); 3336d0060f6SMat Martineau if (!psize) 3346d0060f6SMat Martineau return -EINVAL; 3356d0060f6SMat Martineau 33657040755SPaolo Abeni /* tell the TCP stack to delay the push so that we can safely 33757040755SPaolo Abeni * access the skb after the sendpages call 3386d0060f6SMat Martineau */ 3396d0060f6SMat Martineau ret = do_tcp_sendpages(ssk, pfrag->page, pfrag->offset, psize, 3406d0060f6SMat Martineau msg->msg_flags | MSG_SENDPAGE_NOTLAST); 3416d0060f6SMat Martineau if (ret <= 0) 3426d0060f6SMat Martineau return ret; 3436d0060f6SMat Martineau if (unlikely(ret < psize)) 3446d0060f6SMat Martineau iov_iter_revert(&msg->msg_iter, psize - ret); 3456d0060f6SMat Martineau 34657040755SPaolo Abeni /* if the tail skb extension is still the cached one, collapsing 34757040755SPaolo Abeni * really happened. Note: we can't check for 'same skb' as the sk_buff 34857040755SPaolo Abeni * hdr on tail can be transmitted, freed and re-allocated by the 34957040755SPaolo Abeni * do_tcp_sendpages() call 35057040755SPaolo Abeni */ 35157040755SPaolo Abeni tail = tcp_write_queue_tail(ssk); 35257040755SPaolo Abeni if (mpext && tail && mpext == skb_ext_find(tail, SKB_EXT_MPTCP)) { 35357040755SPaolo Abeni WARN_ON_ONCE(!can_collapse); 35457040755SPaolo Abeni mpext->data_len += ret; 35557040755SPaolo Abeni goto out; 35657040755SPaolo Abeni } 35757040755SPaolo Abeni 3586d0060f6SMat Martineau skb = tcp_write_queue_tail(ssk); 3596d0060f6SMat Martineau mpext = __skb_ext_set(skb, SKB_EXT_MPTCP, msk->cached_ext); 3606d0060f6SMat Martineau msk->cached_ext = NULL; 3616d0060f6SMat Martineau 3626d0060f6SMat Martineau memset(mpext, 0, sizeof(*mpext)); 3636d0060f6SMat Martineau mpext->data_seq = msk->write_seq; 3646d0060f6SMat Martineau mpext->subflow_seq = mptcp_subflow_ctx(ssk)->rel_write_seq; 3656d0060f6SMat Martineau mpext->data_len = ret; 3666d0060f6SMat Martineau mpext->use_map = 1; 3676d0060f6SMat Martineau mpext->dsn64 = 1; 3686d0060f6SMat Martineau 3696d0060f6SMat Martineau pr_debug("data_seq=%llu subflow_seq=%u data_len=%u dsn64=%d", 3706d0060f6SMat Martineau mpext->data_seq, mpext->subflow_seq, mpext->data_len, 3716d0060f6SMat Martineau mpext->dsn64); 3726d0060f6SMat Martineau 37357040755SPaolo Abeni out: 3746d0060f6SMat Martineau pfrag->offset += ret; 3756d0060f6SMat Martineau msk->write_seq += ret; 3766d0060f6SMat Martineau mptcp_subflow_ctx(ssk)->rel_write_seq += ret; 3776d0060f6SMat Martineau 3786d0060f6SMat Martineau return ret; 3796d0060f6SMat Martineau } 3806d0060f6SMat Martineau 381*f296234cSPeter Krystad static struct sock *mptcp_subflow_get_send(struct mptcp_sock *msk) 382*f296234cSPeter Krystad { 383*f296234cSPeter Krystad struct mptcp_subflow_context *subflow; 384*f296234cSPeter Krystad struct sock *backup = NULL; 385*f296234cSPeter Krystad 386*f296234cSPeter Krystad sock_owned_by_me((const struct sock *)msk); 387*f296234cSPeter Krystad 388*f296234cSPeter Krystad mptcp_for_each_subflow(msk, subflow) { 389*f296234cSPeter Krystad struct sock *ssk = mptcp_subflow_tcp_sock(subflow); 390*f296234cSPeter Krystad 391*f296234cSPeter Krystad if (!sk_stream_memory_free(ssk)) { 392*f296234cSPeter Krystad struct socket *sock = ssk->sk_socket; 393*f296234cSPeter Krystad 394*f296234cSPeter Krystad if (sock) { 395*f296234cSPeter Krystad clear_bit(MPTCP_SEND_SPACE, &msk->flags); 396*f296234cSPeter Krystad smp_mb__after_atomic(); 397*f296234cSPeter Krystad 398*f296234cSPeter Krystad /* enables sk->write_space() callbacks */ 399*f296234cSPeter Krystad set_bit(SOCK_NOSPACE, &sock->flags); 400*f296234cSPeter Krystad } 401*f296234cSPeter Krystad 402*f296234cSPeter Krystad return NULL; 403*f296234cSPeter Krystad } 404*f296234cSPeter Krystad 405*f296234cSPeter Krystad if (subflow->backup) { 406*f296234cSPeter Krystad if (!backup) 407*f296234cSPeter Krystad backup = ssk; 408*f296234cSPeter Krystad 409*f296234cSPeter Krystad continue; 410*f296234cSPeter Krystad } 411*f296234cSPeter Krystad 412*f296234cSPeter Krystad return ssk; 413*f296234cSPeter Krystad } 414*f296234cSPeter Krystad 415*f296234cSPeter Krystad return backup; 416*f296234cSPeter Krystad } 417*f296234cSPeter Krystad 4181891c4a0SFlorian Westphal static void ssk_check_wmem(struct mptcp_sock *msk, struct sock *ssk) 4191891c4a0SFlorian Westphal { 4201891c4a0SFlorian Westphal struct socket *sock; 4211891c4a0SFlorian Westphal 4221891c4a0SFlorian Westphal if (likely(sk_stream_is_writeable(ssk))) 4231891c4a0SFlorian Westphal return; 4241891c4a0SFlorian Westphal 4251891c4a0SFlorian Westphal sock = READ_ONCE(ssk->sk_socket); 4261891c4a0SFlorian Westphal 4271891c4a0SFlorian Westphal if (sock) { 4281891c4a0SFlorian Westphal clear_bit(MPTCP_SEND_SPACE, &msk->flags); 4291891c4a0SFlorian Westphal smp_mb__after_atomic(); 4301891c4a0SFlorian Westphal /* set NOSPACE only after clearing SEND_SPACE flag */ 4311891c4a0SFlorian Westphal set_bit(SOCK_NOSPACE, &sock->flags); 4321891c4a0SFlorian Westphal } 4331891c4a0SFlorian Westphal } 4341891c4a0SFlorian Westphal 435f870fa0bSMat Martineau static int mptcp_sendmsg(struct sock *sk, struct msghdr *msg, size_t len) 436f870fa0bSMat Martineau { 43757040755SPaolo Abeni int mss_now = 0, size_goal = 0, ret = 0; 438f870fa0bSMat Martineau struct mptcp_sock *msk = mptcp_sk(sk); 439cec37a6eSPeter Krystad struct socket *ssock; 4406d0060f6SMat Martineau size_t copied = 0; 441cec37a6eSPeter Krystad struct sock *ssk; 4426d0060f6SMat Martineau long timeo; 443f870fa0bSMat Martineau 444f870fa0bSMat Martineau if (msg->msg_flags & ~(MSG_MORE | MSG_DONTWAIT | MSG_NOSIGNAL)) 445f870fa0bSMat Martineau return -EOPNOTSUPP; 446f870fa0bSMat Martineau 447cec37a6eSPeter Krystad lock_sock(sk); 4481954b860SMat Martineau 4491954b860SMat Martineau timeo = sock_sndtimeo(sk, msg->msg_flags & MSG_DONTWAIT); 4501954b860SMat Martineau 4511954b860SMat Martineau if ((1 << sk->sk_state) & ~(TCPF_ESTABLISHED | TCPF_CLOSE_WAIT)) { 4521954b860SMat Martineau ret = sk_stream_wait_connect(sk, &timeo); 4531954b860SMat Martineau if (ret) 4541954b860SMat Martineau goto out; 4551954b860SMat Martineau } 4561954b860SMat Martineau 457cec37a6eSPeter Krystad ssock = __mptcp_tcp_fallback(msk); 4588ab183deSPaolo Abeni if (unlikely(ssock)) { 4598ab183deSPaolo Abeni fallback: 460cec37a6eSPeter Krystad pr_debug("fallback passthrough"); 461cec37a6eSPeter Krystad ret = sock_sendmsg(ssock, msg); 4628ab183deSPaolo Abeni return ret >= 0 ? ret + copied : (copied ? copied : ret); 463cec37a6eSPeter Krystad } 464cec37a6eSPeter Krystad 465*f296234cSPeter Krystad ssk = mptcp_subflow_get_send(msk); 466*f296234cSPeter Krystad while (!sk_stream_memory_free(sk) || !ssk) { 467*f296234cSPeter Krystad ret = sk_stream_wait_memory(sk, &timeo); 468*f296234cSPeter Krystad if (ret) 469*f296234cSPeter Krystad goto out; 470*f296234cSPeter Krystad 471*f296234cSPeter Krystad ssk = mptcp_subflow_get_send(msk); 472*f296234cSPeter Krystad if (list_empty(&msk->conn_list)) { 473*f296234cSPeter Krystad ret = -ENOTCONN; 474*f296234cSPeter Krystad goto out; 475*f296234cSPeter Krystad } 476cec37a6eSPeter Krystad } 477cec37a6eSPeter Krystad 4786d0060f6SMat Martineau pr_debug("conn_list->subflow=%p", ssk); 479cec37a6eSPeter Krystad 4806d0060f6SMat Martineau lock_sock(ssk); 4816d0060f6SMat Martineau while (msg_data_left(msg)) { 48257040755SPaolo Abeni ret = mptcp_sendmsg_frag(sk, ssk, msg, &timeo, &mss_now, 48357040755SPaolo Abeni &size_goal); 4846d0060f6SMat Martineau if (ret < 0) 4856d0060f6SMat Martineau break; 4868ab183deSPaolo Abeni if (ret == 0 && unlikely(__mptcp_needs_tcp_fallback(msk))) { 4878ab183deSPaolo Abeni release_sock(ssk); 4888ab183deSPaolo Abeni ssock = __mptcp_tcp_fallback(msk); 4898ab183deSPaolo Abeni goto fallback; 4908ab183deSPaolo Abeni } 4916d0060f6SMat Martineau 4926d0060f6SMat Martineau copied += ret; 4936d0060f6SMat Martineau } 4946d0060f6SMat Martineau 49557040755SPaolo Abeni if (copied) { 4966d0060f6SMat Martineau ret = copied; 49757040755SPaolo Abeni tcp_push(ssk, msg->msg_flags, mss_now, tcp_sk(ssk)->nonagle, 49857040755SPaolo Abeni size_goal); 49957040755SPaolo Abeni } 5006d0060f6SMat Martineau 5011891c4a0SFlorian Westphal ssk_check_wmem(msk, ssk); 5026d0060f6SMat Martineau release_sock(ssk); 5031954b860SMat Martineau out: 504cec37a6eSPeter Krystad release_sock(sk); 505cec37a6eSPeter Krystad return ret; 506f870fa0bSMat Martineau } 507f870fa0bSMat Martineau 5087a6a6cbcSPaolo Abeni static void mptcp_wait_data(struct sock *sk, long *timeo) 5097a6a6cbcSPaolo Abeni { 5107a6a6cbcSPaolo Abeni DEFINE_WAIT_FUNC(wait, woken_wake_function); 5117a6a6cbcSPaolo Abeni struct mptcp_sock *msk = mptcp_sk(sk); 5127a6a6cbcSPaolo Abeni 5137a6a6cbcSPaolo Abeni add_wait_queue(sk_sleep(sk), &wait); 5147a6a6cbcSPaolo Abeni sk_set_bit(SOCKWQ_ASYNC_WAITDATA, sk); 5157a6a6cbcSPaolo Abeni 5167a6a6cbcSPaolo Abeni sk_wait_event(sk, timeo, 5177a6a6cbcSPaolo Abeni test_and_clear_bit(MPTCP_DATA_READY, &msk->flags), &wait); 5187a6a6cbcSPaolo Abeni 5197a6a6cbcSPaolo Abeni sk_clear_bit(SOCKWQ_ASYNC_WAITDATA, sk); 5207a6a6cbcSPaolo Abeni remove_wait_queue(sk_sleep(sk), &wait); 5217a6a6cbcSPaolo Abeni } 5227a6a6cbcSPaolo Abeni 5236771bfd9SFlorian Westphal static int __mptcp_recvmsg_mskq(struct mptcp_sock *msk, 5246771bfd9SFlorian Westphal struct msghdr *msg, 5256771bfd9SFlorian Westphal size_t len) 5266771bfd9SFlorian Westphal { 5276771bfd9SFlorian Westphal struct sock *sk = (struct sock *)msk; 5286771bfd9SFlorian Westphal struct sk_buff *skb; 5296771bfd9SFlorian Westphal int copied = 0; 5306771bfd9SFlorian Westphal 5316771bfd9SFlorian Westphal while ((skb = skb_peek(&sk->sk_receive_queue)) != NULL) { 5326771bfd9SFlorian Westphal u32 offset = MPTCP_SKB_CB(skb)->offset; 5336771bfd9SFlorian Westphal u32 data_len = skb->len - offset; 5346771bfd9SFlorian Westphal u32 count = min_t(size_t, len - copied, data_len); 5356771bfd9SFlorian Westphal int err; 5366771bfd9SFlorian Westphal 5376771bfd9SFlorian Westphal err = skb_copy_datagram_msg(skb, offset, msg, count); 5386771bfd9SFlorian Westphal if (unlikely(err < 0)) { 5396771bfd9SFlorian Westphal if (!copied) 5406771bfd9SFlorian Westphal return err; 5416771bfd9SFlorian Westphal break; 5426771bfd9SFlorian Westphal } 5436771bfd9SFlorian Westphal 5446771bfd9SFlorian Westphal copied += count; 5456771bfd9SFlorian Westphal 5466771bfd9SFlorian Westphal if (count < data_len) { 5476771bfd9SFlorian Westphal MPTCP_SKB_CB(skb)->offset += count; 5486771bfd9SFlorian Westphal break; 5496771bfd9SFlorian Westphal } 5506771bfd9SFlorian Westphal 5516771bfd9SFlorian Westphal __skb_unlink(skb, &sk->sk_receive_queue); 5526771bfd9SFlorian Westphal __kfree_skb(skb); 5536771bfd9SFlorian Westphal 5546771bfd9SFlorian Westphal if (copied >= len) 5556771bfd9SFlorian Westphal break; 5566771bfd9SFlorian Westphal } 5576771bfd9SFlorian Westphal 5586771bfd9SFlorian Westphal return copied; 5596771bfd9SFlorian Westphal } 5606771bfd9SFlorian Westphal 5616771bfd9SFlorian Westphal static bool __mptcp_move_skbs(struct mptcp_sock *msk) 5626771bfd9SFlorian Westphal { 5636771bfd9SFlorian Westphal unsigned int moved = 0; 5646771bfd9SFlorian Westphal bool done; 5656771bfd9SFlorian Westphal 5666771bfd9SFlorian Westphal do { 5676771bfd9SFlorian Westphal struct sock *ssk = mptcp_subflow_recv_lookup(msk); 5686771bfd9SFlorian Westphal 5696771bfd9SFlorian Westphal if (!ssk) 5706771bfd9SFlorian Westphal break; 5716771bfd9SFlorian Westphal 5726771bfd9SFlorian Westphal lock_sock(ssk); 5736771bfd9SFlorian Westphal done = __mptcp_move_skbs_from_subflow(msk, ssk, &moved); 5746771bfd9SFlorian Westphal release_sock(ssk); 5756771bfd9SFlorian Westphal } while (!done); 5766771bfd9SFlorian Westphal 5776771bfd9SFlorian Westphal return moved > 0; 5786771bfd9SFlorian Westphal } 5796771bfd9SFlorian Westphal 580f870fa0bSMat Martineau static int mptcp_recvmsg(struct sock *sk, struct msghdr *msg, size_t len, 581f870fa0bSMat Martineau int nonblock, int flags, int *addr_len) 582f870fa0bSMat Martineau { 583f870fa0bSMat Martineau struct mptcp_sock *msk = mptcp_sk(sk); 584cec37a6eSPeter Krystad struct socket *ssock; 585cec37a6eSPeter Krystad int copied = 0; 5867a6a6cbcSPaolo Abeni int target; 5877a6a6cbcSPaolo Abeni long timeo; 588f870fa0bSMat Martineau 589f870fa0bSMat Martineau if (msg->msg_flags & ~(MSG_WAITALL | MSG_DONTWAIT)) 590f870fa0bSMat Martineau return -EOPNOTSUPP; 591f870fa0bSMat Martineau 592cec37a6eSPeter Krystad lock_sock(sk); 593cec37a6eSPeter Krystad ssock = __mptcp_tcp_fallback(msk); 5948ab183deSPaolo Abeni if (unlikely(ssock)) { 5958ab183deSPaolo Abeni fallback: 596cec37a6eSPeter Krystad pr_debug("fallback-read subflow=%p", 597cec37a6eSPeter Krystad mptcp_subflow_ctx(ssock->sk)); 598cec37a6eSPeter Krystad copied = sock_recvmsg(ssock, msg, flags); 599cec37a6eSPeter Krystad return copied; 600cec37a6eSPeter Krystad } 601cec37a6eSPeter Krystad 6027a6a6cbcSPaolo Abeni timeo = sock_rcvtimeo(sk, nonblock); 6037a6a6cbcSPaolo Abeni 6047a6a6cbcSPaolo Abeni len = min_t(size_t, len, INT_MAX); 6057a6a6cbcSPaolo Abeni target = sock_rcvlowat(sk, flags & MSG_WAITALL, len); 6067a6a6cbcSPaolo Abeni 6076771bfd9SFlorian Westphal while (len > (size_t)copied) { 6087a6a6cbcSPaolo Abeni int bytes_read; 6097a6a6cbcSPaolo Abeni 6106771bfd9SFlorian Westphal bytes_read = __mptcp_recvmsg_mskq(msk, msg, len - copied); 6116771bfd9SFlorian Westphal if (unlikely(bytes_read < 0)) { 6127a6a6cbcSPaolo Abeni if (!copied) 6137a6a6cbcSPaolo Abeni copied = bytes_read; 6146771bfd9SFlorian Westphal goto out_err; 615cec37a6eSPeter Krystad } 616cec37a6eSPeter Krystad 6177a6a6cbcSPaolo Abeni copied += bytes_read; 6187a6a6cbcSPaolo Abeni 6196771bfd9SFlorian Westphal if (skb_queue_empty(&sk->sk_receive_queue) && 6206771bfd9SFlorian Westphal __mptcp_move_skbs(msk)) 6216771bfd9SFlorian Westphal continue; 6227a6a6cbcSPaolo Abeni 6237a6a6cbcSPaolo Abeni /* only the master socket status is relevant here. The exit 6247a6a6cbcSPaolo Abeni * conditions mirror closely tcp_recvmsg() 6257a6a6cbcSPaolo Abeni */ 6267a6a6cbcSPaolo Abeni if (copied >= target) 6277a6a6cbcSPaolo Abeni break; 6287a6a6cbcSPaolo Abeni 6297a6a6cbcSPaolo Abeni if (copied) { 6307a6a6cbcSPaolo Abeni if (sk->sk_err || 6317a6a6cbcSPaolo Abeni sk->sk_state == TCP_CLOSE || 6327a6a6cbcSPaolo Abeni (sk->sk_shutdown & RCV_SHUTDOWN) || 6337a6a6cbcSPaolo Abeni !timeo || 6347a6a6cbcSPaolo Abeni signal_pending(current)) 6357a6a6cbcSPaolo Abeni break; 6367a6a6cbcSPaolo Abeni } else { 6377a6a6cbcSPaolo Abeni if (sk->sk_err) { 6387a6a6cbcSPaolo Abeni copied = sock_error(sk); 6397a6a6cbcSPaolo Abeni break; 6407a6a6cbcSPaolo Abeni } 6417a6a6cbcSPaolo Abeni 6427a6a6cbcSPaolo Abeni if (sk->sk_shutdown & RCV_SHUTDOWN) 6437a6a6cbcSPaolo Abeni break; 6447a6a6cbcSPaolo Abeni 6457a6a6cbcSPaolo Abeni if (sk->sk_state == TCP_CLOSE) { 6467a6a6cbcSPaolo Abeni copied = -ENOTCONN; 6477a6a6cbcSPaolo Abeni break; 6487a6a6cbcSPaolo Abeni } 6497a6a6cbcSPaolo Abeni 6507a6a6cbcSPaolo Abeni if (!timeo) { 6517a6a6cbcSPaolo Abeni copied = -EAGAIN; 6527a6a6cbcSPaolo Abeni break; 6537a6a6cbcSPaolo Abeni } 6547a6a6cbcSPaolo Abeni 6557a6a6cbcSPaolo Abeni if (signal_pending(current)) { 6567a6a6cbcSPaolo Abeni copied = sock_intr_errno(timeo); 6577a6a6cbcSPaolo Abeni break; 6587a6a6cbcSPaolo Abeni } 6597a6a6cbcSPaolo Abeni } 6607a6a6cbcSPaolo Abeni 6617a6a6cbcSPaolo Abeni pr_debug("block timeout %ld", timeo); 6627a6a6cbcSPaolo Abeni mptcp_wait_data(sk, &timeo); 6638ab183deSPaolo Abeni if (unlikely(__mptcp_tcp_fallback(msk))) 6648ab183deSPaolo Abeni goto fallback; 6657a6a6cbcSPaolo Abeni } 6667a6a6cbcSPaolo Abeni 6676771bfd9SFlorian Westphal if (skb_queue_empty(&sk->sk_receive_queue)) { 6686771bfd9SFlorian Westphal /* entire backlog drained, clear DATA_READY. */ 6697a6a6cbcSPaolo Abeni clear_bit(MPTCP_DATA_READY, &msk->flags); 6707a6a6cbcSPaolo Abeni 6716771bfd9SFlorian Westphal /* .. race-breaker: ssk might have gotten new data 6726771bfd9SFlorian Westphal * after last __mptcp_move_skbs() returned false. 6737a6a6cbcSPaolo Abeni */ 6746771bfd9SFlorian Westphal if (unlikely(__mptcp_move_skbs(msk))) 6756771bfd9SFlorian Westphal set_bit(MPTCP_DATA_READY, &msk->flags); 6766771bfd9SFlorian Westphal } else if (unlikely(!test_bit(MPTCP_DATA_READY, &msk->flags))) { 6776771bfd9SFlorian Westphal /* data to read but mptcp_wait_data() cleared DATA_READY */ 6787a6a6cbcSPaolo Abeni set_bit(MPTCP_DATA_READY, &msk->flags); 6797a6a6cbcSPaolo Abeni } 6806771bfd9SFlorian Westphal out_err: 681cec37a6eSPeter Krystad release_sock(sk); 682cec37a6eSPeter Krystad return copied; 683cec37a6eSPeter Krystad } 684cec37a6eSPeter Krystad 685cec37a6eSPeter Krystad /* subflow sockets can be either outgoing (connect) or incoming 686cec37a6eSPeter Krystad * (accept). 687cec37a6eSPeter Krystad * 688cec37a6eSPeter Krystad * Outgoing subflows use in-kernel sockets. 689cec37a6eSPeter Krystad * Incoming subflows do not have their own 'struct socket' allocated, 690cec37a6eSPeter Krystad * so we need to use tcp_close() after detaching them from the mptcp 691cec37a6eSPeter Krystad * parent socket. 692cec37a6eSPeter Krystad */ 693cec37a6eSPeter Krystad static void __mptcp_close_ssk(struct sock *sk, struct sock *ssk, 694cec37a6eSPeter Krystad struct mptcp_subflow_context *subflow, 695cec37a6eSPeter Krystad long timeout) 696cec37a6eSPeter Krystad { 697cec37a6eSPeter Krystad struct socket *sock = READ_ONCE(ssk->sk_socket); 698cec37a6eSPeter Krystad 699cec37a6eSPeter Krystad list_del(&subflow->node); 700cec37a6eSPeter Krystad 701cec37a6eSPeter Krystad if (sock && sock != sk->sk_socket) { 702cec37a6eSPeter Krystad /* outgoing subflow */ 703cec37a6eSPeter Krystad sock_release(sock); 704cec37a6eSPeter Krystad } else { 705cec37a6eSPeter Krystad /* incoming subflow */ 706cec37a6eSPeter Krystad tcp_close(ssk, timeout); 707cec37a6eSPeter Krystad } 708f870fa0bSMat Martineau } 709f870fa0bSMat Martineau 710dc24f8b4SPaolo Abeni static unsigned int mptcp_sync_mss(struct sock *sk, u32 pmtu) 711dc24f8b4SPaolo Abeni { 712dc24f8b4SPaolo Abeni return 0; 713dc24f8b4SPaolo Abeni } 714dc24f8b4SPaolo Abeni 71580992017SPaolo Abeni static void mptcp_worker(struct work_struct *work) 71680992017SPaolo Abeni { 71780992017SPaolo Abeni struct mptcp_sock *msk = container_of(work, struct mptcp_sock, work); 71880992017SPaolo Abeni struct sock *sk = &msk->sk.icsk_inet.sk; 71980992017SPaolo Abeni 72080992017SPaolo Abeni lock_sock(sk); 7216771bfd9SFlorian Westphal __mptcp_move_skbs(msk); 72280992017SPaolo Abeni release_sock(sk); 72380992017SPaolo Abeni sock_put(sk); 72480992017SPaolo Abeni } 72580992017SPaolo Abeni 726784325e9SMatthieu Baerts static int __mptcp_init_sock(struct sock *sk) 727f870fa0bSMat Martineau { 728cec37a6eSPeter Krystad struct mptcp_sock *msk = mptcp_sk(sk); 729cec37a6eSPeter Krystad 730cec37a6eSPeter Krystad INIT_LIST_HEAD(&msk->conn_list); 7311891c4a0SFlorian Westphal __set_bit(MPTCP_SEND_SPACE, &msk->flags); 73280992017SPaolo Abeni INIT_WORK(&msk->work, mptcp_worker); 733cec37a6eSPeter Krystad 7348ab183deSPaolo Abeni msk->first = NULL; 735dc24f8b4SPaolo Abeni inet_csk(sk)->icsk_sync_mss = mptcp_sync_mss; 7368ab183deSPaolo Abeni 7371b1c7a0eSPeter Krystad mptcp_pm_data_init(msk); 7381b1c7a0eSPeter Krystad 739f870fa0bSMat Martineau return 0; 740f870fa0bSMat Martineau } 741f870fa0bSMat Martineau 742784325e9SMatthieu Baerts static int mptcp_init_sock(struct sock *sk) 743784325e9SMatthieu Baerts { 744784325e9SMatthieu Baerts if (!mptcp_is_enabled(sock_net(sk))) 745784325e9SMatthieu Baerts return -ENOPROTOOPT; 746784325e9SMatthieu Baerts 747784325e9SMatthieu Baerts return __mptcp_init_sock(sk); 748784325e9SMatthieu Baerts } 749784325e9SMatthieu Baerts 75080992017SPaolo Abeni static void mptcp_cancel_work(struct sock *sk) 75180992017SPaolo Abeni { 75280992017SPaolo Abeni struct mptcp_sock *msk = mptcp_sk(sk); 75380992017SPaolo Abeni 75480992017SPaolo Abeni if (cancel_work_sync(&msk->work)) 75580992017SPaolo Abeni sock_put(sk); 75680992017SPaolo Abeni } 75780992017SPaolo Abeni 75876c42a29SMat Martineau static void mptcp_subflow_shutdown(struct sock *ssk, int how, 75976c42a29SMat Martineau bool data_fin_tx_enable, u64 data_fin_tx_seq) 76021498490SPeter Krystad { 76121498490SPeter Krystad lock_sock(ssk); 76221498490SPeter Krystad 76321498490SPeter Krystad switch (ssk->sk_state) { 76421498490SPeter Krystad case TCP_LISTEN: 76521498490SPeter Krystad if (!(how & RCV_SHUTDOWN)) 76621498490SPeter Krystad break; 76721498490SPeter Krystad /* fall through */ 76821498490SPeter Krystad case TCP_SYN_SENT: 76921498490SPeter Krystad tcp_disconnect(ssk, O_NONBLOCK); 77021498490SPeter Krystad break; 77121498490SPeter Krystad default: 77276c42a29SMat Martineau if (data_fin_tx_enable) { 77376c42a29SMat Martineau struct mptcp_subflow_context *subflow; 77476c42a29SMat Martineau 77576c42a29SMat Martineau subflow = mptcp_subflow_ctx(ssk); 77676c42a29SMat Martineau subflow->data_fin_tx_seq = data_fin_tx_seq; 77776c42a29SMat Martineau subflow->data_fin_tx_enable = 1; 77876c42a29SMat Martineau } 77976c42a29SMat Martineau 78021498490SPeter Krystad ssk->sk_shutdown |= how; 78121498490SPeter Krystad tcp_shutdown(ssk, how); 78221498490SPeter Krystad break; 78321498490SPeter Krystad } 78421498490SPeter Krystad 78521498490SPeter Krystad /* Wake up anyone sleeping in poll. */ 78621498490SPeter Krystad ssk->sk_state_change(ssk); 78721498490SPeter Krystad release_sock(ssk); 78821498490SPeter Krystad } 78921498490SPeter Krystad 7908ab183deSPaolo Abeni /* Called with msk lock held, releases such lock before returning */ 7912c22c06cSFlorian Westphal static void mptcp_close(struct sock *sk, long timeout) 792f870fa0bSMat Martineau { 793cec37a6eSPeter Krystad struct mptcp_subflow_context *subflow, *tmp; 794f870fa0bSMat Martineau struct mptcp_sock *msk = mptcp_sk(sk); 795b2c5b614SFlorian Westphal LIST_HEAD(conn_list); 79676c42a29SMat Martineau u64 data_fin_tx_seq; 797f870fa0bSMat Martineau 7982c22c06cSFlorian Westphal lock_sock(sk); 7992c22c06cSFlorian Westphal 80079c0949eSPeter Krystad mptcp_token_destroy(msk->token); 801f870fa0bSMat Martineau inet_sk_state_store(sk, TCP_CLOSE); 802f870fa0bSMat Martineau 803b2c5b614SFlorian Westphal list_splice_init(&msk->conn_list, &conn_list); 804b2c5b614SFlorian Westphal 80576c42a29SMat Martineau data_fin_tx_seq = msk->write_seq; 80676c42a29SMat Martineau 807b2c5b614SFlorian Westphal release_sock(sk); 808b2c5b614SFlorian Westphal 809b2c5b614SFlorian Westphal list_for_each_entry_safe(subflow, tmp, &conn_list, node) { 810cec37a6eSPeter Krystad struct sock *ssk = mptcp_subflow_tcp_sock(subflow); 811cec37a6eSPeter Krystad 81276c42a29SMat Martineau subflow->data_fin_tx_seq = data_fin_tx_seq; 81376c42a29SMat Martineau subflow->data_fin_tx_enable = 1; 814cec37a6eSPeter Krystad __mptcp_close_ssk(sk, ssk, subflow, timeout); 815f870fa0bSMat Martineau } 816f870fa0bSMat Martineau 81780992017SPaolo Abeni mptcp_cancel_work(sk); 81880992017SPaolo Abeni 8196771bfd9SFlorian Westphal __skb_queue_purge(&sk->sk_receive_queue); 8206771bfd9SFlorian Westphal 821cec37a6eSPeter Krystad sk_common_release(sk); 822f870fa0bSMat Martineau } 823f870fa0bSMat Martineau 824cf7da0d6SPeter Krystad static void mptcp_copy_inaddrs(struct sock *msk, const struct sock *ssk) 825cf7da0d6SPeter Krystad { 826cf7da0d6SPeter Krystad #if IS_ENABLED(CONFIG_MPTCP_IPV6) 827cf7da0d6SPeter Krystad const struct ipv6_pinfo *ssk6 = inet6_sk(ssk); 828cf7da0d6SPeter Krystad struct ipv6_pinfo *msk6 = inet6_sk(msk); 829cf7da0d6SPeter Krystad 830cf7da0d6SPeter Krystad msk->sk_v6_daddr = ssk->sk_v6_daddr; 831cf7da0d6SPeter Krystad msk->sk_v6_rcv_saddr = ssk->sk_v6_rcv_saddr; 832cf7da0d6SPeter Krystad 833cf7da0d6SPeter Krystad if (msk6 && ssk6) { 834cf7da0d6SPeter Krystad msk6->saddr = ssk6->saddr; 835cf7da0d6SPeter Krystad msk6->flow_label = ssk6->flow_label; 836cf7da0d6SPeter Krystad } 837cf7da0d6SPeter Krystad #endif 838cf7da0d6SPeter Krystad 839cf7da0d6SPeter Krystad inet_sk(msk)->inet_num = inet_sk(ssk)->inet_num; 840cf7da0d6SPeter Krystad inet_sk(msk)->inet_dport = inet_sk(ssk)->inet_dport; 841cf7da0d6SPeter Krystad inet_sk(msk)->inet_sport = inet_sk(ssk)->inet_sport; 842cf7da0d6SPeter Krystad inet_sk(msk)->inet_daddr = inet_sk(ssk)->inet_daddr; 843cf7da0d6SPeter Krystad inet_sk(msk)->inet_saddr = inet_sk(ssk)->inet_saddr; 844cf7da0d6SPeter Krystad inet_sk(msk)->inet_rcv_saddr = inet_sk(ssk)->inet_rcv_saddr; 845cf7da0d6SPeter Krystad } 846cf7da0d6SPeter Krystad 847b0519de8SFlorian Westphal #if IS_ENABLED(CONFIG_MPTCP_IPV6) 848b0519de8SFlorian Westphal static struct ipv6_pinfo *mptcp_inet6_sk(const struct sock *sk) 849b0519de8SFlorian Westphal { 850b0519de8SFlorian Westphal unsigned int offset = sizeof(struct mptcp6_sock) - sizeof(struct ipv6_pinfo); 851b0519de8SFlorian Westphal 852b0519de8SFlorian Westphal return (struct ipv6_pinfo *)(((u8 *)sk) + offset); 853b0519de8SFlorian Westphal } 854b0519de8SFlorian Westphal #endif 855b0519de8SFlorian Westphal 85658b09919SPaolo Abeni struct sock *mptcp_sk_clone(const struct sock *sk, struct request_sock *req) 857b0519de8SFlorian Westphal { 85858b09919SPaolo Abeni struct mptcp_subflow_request_sock *subflow_req = mptcp_subflow_rsk(req); 859b0519de8SFlorian Westphal struct sock *nsk = sk_clone_lock(sk, GFP_ATOMIC); 86058b09919SPaolo Abeni struct mptcp_sock *msk; 86158b09919SPaolo Abeni u64 ack_seq; 862b0519de8SFlorian Westphal 863b0519de8SFlorian Westphal if (!nsk) 864b0519de8SFlorian Westphal return NULL; 865b0519de8SFlorian Westphal 866b0519de8SFlorian Westphal #if IS_ENABLED(CONFIG_MPTCP_IPV6) 867b0519de8SFlorian Westphal if (nsk->sk_family == AF_INET6) 868b0519de8SFlorian Westphal inet_sk(nsk)->pinet6 = mptcp_inet6_sk(nsk); 869b0519de8SFlorian Westphal #endif 870b0519de8SFlorian Westphal 87158b09919SPaolo Abeni __mptcp_init_sock(nsk); 87258b09919SPaolo Abeni 87358b09919SPaolo Abeni msk = mptcp_sk(nsk); 87458b09919SPaolo Abeni msk->local_key = subflow_req->local_key; 87558b09919SPaolo Abeni msk->token = subflow_req->token; 87658b09919SPaolo Abeni msk->subflow = NULL; 87758b09919SPaolo Abeni 87858b09919SPaolo Abeni if (unlikely(mptcp_token_new_accept(subflow_req->token, nsk))) { 87958b09919SPaolo Abeni bh_unlock_sock(nsk); 88058b09919SPaolo Abeni 88158b09919SPaolo Abeni /* we can't call into mptcp_close() here - possible BH context 88258b09919SPaolo Abeni * free the sock directly 88358b09919SPaolo Abeni */ 88458b09919SPaolo Abeni nsk->sk_prot->destroy(nsk); 88558b09919SPaolo Abeni sk_free(nsk); 88658b09919SPaolo Abeni return NULL; 88758b09919SPaolo Abeni } 88858b09919SPaolo Abeni 88958b09919SPaolo Abeni msk->write_seq = subflow_req->idsn + 1; 89058b09919SPaolo Abeni if (subflow_req->remote_key_valid) { 89158b09919SPaolo Abeni msk->can_ack = true; 89258b09919SPaolo Abeni msk->remote_key = subflow_req->remote_key; 89358b09919SPaolo Abeni mptcp_crypto_key_sha(msk->remote_key, NULL, &ack_seq); 89458b09919SPaolo Abeni ack_seq++; 89558b09919SPaolo Abeni msk->ack_seq = ack_seq; 89658b09919SPaolo Abeni } 8977f20d5fcSPaolo Abeni 8987f20d5fcSPaolo Abeni /* will be fully established after successful MPC subflow creation */ 8997f20d5fcSPaolo Abeni inet_sk_state_store(nsk, TCP_SYN_RECV); 90058b09919SPaolo Abeni bh_unlock_sock(nsk); 90158b09919SPaolo Abeni 90258b09919SPaolo Abeni /* keep a single reference */ 90358b09919SPaolo Abeni __sock_put(nsk); 904b0519de8SFlorian Westphal return nsk; 905b0519de8SFlorian Westphal } 906b0519de8SFlorian Westphal 907cf7da0d6SPeter Krystad static struct sock *mptcp_accept(struct sock *sk, int flags, int *err, 908cf7da0d6SPeter Krystad bool kern) 909cf7da0d6SPeter Krystad { 910cf7da0d6SPeter Krystad struct mptcp_sock *msk = mptcp_sk(sk); 911cf7da0d6SPeter Krystad struct socket *listener; 912cf7da0d6SPeter Krystad struct sock *newsk; 913cf7da0d6SPeter Krystad 914cf7da0d6SPeter Krystad listener = __mptcp_nmpc_socket(msk); 915cf7da0d6SPeter Krystad if (WARN_ON_ONCE(!listener)) { 916cf7da0d6SPeter Krystad *err = -EINVAL; 917cf7da0d6SPeter Krystad return NULL; 918cf7da0d6SPeter Krystad } 919cf7da0d6SPeter Krystad 920cf7da0d6SPeter Krystad pr_debug("msk=%p, listener=%p", msk, mptcp_subflow_ctx(listener->sk)); 921cf7da0d6SPeter Krystad newsk = inet_csk_accept(listener->sk, flags, err, kern); 922cf7da0d6SPeter Krystad if (!newsk) 923cf7da0d6SPeter Krystad return NULL; 924cf7da0d6SPeter Krystad 925cf7da0d6SPeter Krystad pr_debug("msk=%p, subflow is mptcp=%d", msk, sk_is_mptcp(newsk)); 926cf7da0d6SPeter Krystad 927cf7da0d6SPeter Krystad if (sk_is_mptcp(newsk)) { 928cf7da0d6SPeter Krystad struct mptcp_subflow_context *subflow; 929cf7da0d6SPeter Krystad struct sock *new_mptcp_sock; 930cf7da0d6SPeter Krystad struct sock *ssk = newsk; 931cf7da0d6SPeter Krystad 932cf7da0d6SPeter Krystad subflow = mptcp_subflow_ctx(newsk); 93358b09919SPaolo Abeni new_mptcp_sock = subflow->conn; 93458b09919SPaolo Abeni 93558b09919SPaolo Abeni /* is_mptcp should be false if subflow->conn is missing, see 93658b09919SPaolo Abeni * subflow_syn_recv_sock() 93758b09919SPaolo Abeni */ 93858b09919SPaolo Abeni if (WARN_ON_ONCE(!new_mptcp_sock)) { 93958b09919SPaolo Abeni tcp_sk(newsk)->is_mptcp = 0; 94058b09919SPaolo Abeni return newsk; 94158b09919SPaolo Abeni } 94258b09919SPaolo Abeni 94358b09919SPaolo Abeni /* acquire the 2nd reference for the owning socket */ 94458b09919SPaolo Abeni sock_hold(new_mptcp_sock); 945cf7da0d6SPeter Krystad 946cf7da0d6SPeter Krystad local_bh_disable(); 94758b09919SPaolo Abeni bh_lock_sock(new_mptcp_sock); 948cf7da0d6SPeter Krystad msk = mptcp_sk(new_mptcp_sock); 9498ab183deSPaolo Abeni msk->first = newsk; 950cf7da0d6SPeter Krystad 951cf7da0d6SPeter Krystad newsk = new_mptcp_sock; 952cf7da0d6SPeter Krystad mptcp_copy_inaddrs(newsk, ssk); 953cf7da0d6SPeter Krystad list_add(&subflow->node, &msk->conn_list); 954cf7da0d6SPeter Krystad 955cf7da0d6SPeter Krystad bh_unlock_sock(new_mptcp_sock); 956cf7da0d6SPeter Krystad local_bh_enable(); 957cf7da0d6SPeter Krystad } 958cf7da0d6SPeter Krystad 959cf7da0d6SPeter Krystad return newsk; 960cf7da0d6SPeter Krystad } 961cf7da0d6SPeter Krystad 96279c0949eSPeter Krystad static void mptcp_destroy(struct sock *sk) 96379c0949eSPeter Krystad { 964c9fd9c5fSFlorian Westphal struct mptcp_sock *msk = mptcp_sk(sk); 965c9fd9c5fSFlorian Westphal 966c9fd9c5fSFlorian Westphal if (msk->cached_ext) 967c9fd9c5fSFlorian Westphal __skb_ext_put(msk->cached_ext); 96879c0949eSPeter Krystad } 96979c0949eSPeter Krystad 970717e79c8SPeter Krystad static int mptcp_setsockopt(struct sock *sk, int level, int optname, 97150e741bbSFlorian Westphal char __user *optval, unsigned int optlen) 972717e79c8SPeter Krystad { 973717e79c8SPeter Krystad struct mptcp_sock *msk = mptcp_sk(sk); 974717e79c8SPeter Krystad struct socket *ssock; 975717e79c8SPeter Krystad 976717e79c8SPeter Krystad pr_debug("msk=%p", msk); 977717e79c8SPeter Krystad 978717e79c8SPeter Krystad /* @@ the meaning of setsockopt() when the socket is connected and 979b6e4a1aeSMat Martineau * there are multiple subflows is not yet defined. It is up to the 980b6e4a1aeSMat Martineau * MPTCP-level socket to configure the subflows until the subflow 981b6e4a1aeSMat Martineau * is in TCP fallback, when TCP socket options are passed through 982b6e4a1aeSMat Martineau * to the one remaining subflow. 983717e79c8SPeter Krystad */ 984717e79c8SPeter Krystad lock_sock(sk); 985b6e4a1aeSMat Martineau ssock = __mptcp_tcp_fallback(msk); 986b6e4a1aeSMat Martineau if (ssock) 987b6e4a1aeSMat Martineau return tcp_setsockopt(ssock->sk, level, optname, optval, 988b6e4a1aeSMat Martineau optlen); 98950e741bbSFlorian Westphal 99050e741bbSFlorian Westphal release_sock(sk); 99150e741bbSFlorian Westphal 992b6e4a1aeSMat Martineau return -EOPNOTSUPP; 993717e79c8SPeter Krystad } 994717e79c8SPeter Krystad 995717e79c8SPeter Krystad static int mptcp_getsockopt(struct sock *sk, int level, int optname, 99650e741bbSFlorian Westphal char __user *optval, int __user *option) 997717e79c8SPeter Krystad { 998717e79c8SPeter Krystad struct mptcp_sock *msk = mptcp_sk(sk); 999717e79c8SPeter Krystad struct socket *ssock; 1000717e79c8SPeter Krystad 1001717e79c8SPeter Krystad pr_debug("msk=%p", msk); 1002717e79c8SPeter Krystad 1003b6e4a1aeSMat Martineau /* @@ the meaning of setsockopt() when the socket is connected and 1004b6e4a1aeSMat Martineau * there are multiple subflows is not yet defined. It is up to the 1005b6e4a1aeSMat Martineau * MPTCP-level socket to configure the subflows until the subflow 1006b6e4a1aeSMat Martineau * is in TCP fallback, when socket options are passed through 1007b6e4a1aeSMat Martineau * to the one remaining subflow. 1008717e79c8SPeter Krystad */ 1009717e79c8SPeter Krystad lock_sock(sk); 1010b6e4a1aeSMat Martineau ssock = __mptcp_tcp_fallback(msk); 1011b6e4a1aeSMat Martineau if (ssock) 1012b6e4a1aeSMat Martineau return tcp_getsockopt(ssock->sk, level, optname, optval, 1013b6e4a1aeSMat Martineau option); 101450e741bbSFlorian Westphal 101550e741bbSFlorian Westphal release_sock(sk); 101650e741bbSFlorian Westphal 1017b6e4a1aeSMat Martineau return -EOPNOTSUPP; 1018717e79c8SPeter Krystad } 1019717e79c8SPeter Krystad 102014c441b5SPaolo Abeni #define MPTCP_DEFERRED_ALL TCPF_DELACK_TIMER_DEFERRED 102114c441b5SPaolo Abeni 102214c441b5SPaolo Abeni /* this is very alike tcp_release_cb() but we must handle differently a 102314c441b5SPaolo Abeni * different set of events 102414c441b5SPaolo Abeni */ 102514c441b5SPaolo Abeni static void mptcp_release_cb(struct sock *sk) 102614c441b5SPaolo Abeni { 102714c441b5SPaolo Abeni unsigned long flags, nflags; 102814c441b5SPaolo Abeni 102914c441b5SPaolo Abeni do { 103014c441b5SPaolo Abeni flags = sk->sk_tsq_flags; 103114c441b5SPaolo Abeni if (!(flags & MPTCP_DEFERRED_ALL)) 103214c441b5SPaolo Abeni return; 103314c441b5SPaolo Abeni nflags = flags & ~MPTCP_DEFERRED_ALL; 103414c441b5SPaolo Abeni } while (cmpxchg(&sk->sk_tsq_flags, flags, nflags) != flags); 103514c441b5SPaolo Abeni 103614c441b5SPaolo Abeni if (flags & TCPF_DELACK_TIMER_DEFERRED) { 103714c441b5SPaolo Abeni struct mptcp_sock *msk = mptcp_sk(sk); 103814c441b5SPaolo Abeni struct sock *ssk; 103914c441b5SPaolo Abeni 104014c441b5SPaolo Abeni ssk = mptcp_subflow_recv_lookup(msk); 104114c441b5SPaolo Abeni if (!ssk || !schedule_work(&msk->work)) 104214c441b5SPaolo Abeni __sock_put(sk); 104314c441b5SPaolo Abeni } 104414c441b5SPaolo Abeni } 104514c441b5SPaolo Abeni 1046cec37a6eSPeter Krystad static int mptcp_get_port(struct sock *sk, unsigned short snum) 1047f870fa0bSMat Martineau { 1048f870fa0bSMat Martineau struct mptcp_sock *msk = mptcp_sk(sk); 1049cec37a6eSPeter Krystad struct socket *ssock; 1050f870fa0bSMat Martineau 1051cec37a6eSPeter Krystad ssock = __mptcp_nmpc_socket(msk); 1052cec37a6eSPeter Krystad pr_debug("msk=%p, subflow=%p", msk, ssock); 1053cec37a6eSPeter Krystad if (WARN_ON_ONCE(!ssock)) 1054cec37a6eSPeter Krystad return -EINVAL; 1055f870fa0bSMat Martineau 1056cec37a6eSPeter Krystad return inet_csk_get_port(ssock->sk, snum); 1057cec37a6eSPeter Krystad } 1058f870fa0bSMat Martineau 1059cec37a6eSPeter Krystad void mptcp_finish_connect(struct sock *ssk) 1060cec37a6eSPeter Krystad { 1061cec37a6eSPeter Krystad struct mptcp_subflow_context *subflow; 1062cec37a6eSPeter Krystad struct mptcp_sock *msk; 1063cec37a6eSPeter Krystad struct sock *sk; 10646d0060f6SMat Martineau u64 ack_seq; 1065f870fa0bSMat Martineau 1066cec37a6eSPeter Krystad subflow = mptcp_subflow_ctx(ssk); 1067f870fa0bSMat Martineau 1068cec37a6eSPeter Krystad if (!subflow->mp_capable) 1069cec37a6eSPeter Krystad return; 1070cec37a6eSPeter Krystad 1071cec37a6eSPeter Krystad sk = subflow->conn; 1072cec37a6eSPeter Krystad msk = mptcp_sk(sk); 1073cec37a6eSPeter Krystad 1074648ef4b8SMat Martineau pr_debug("msk=%p, token=%u", sk, subflow->token); 1075648ef4b8SMat Martineau 10766d0060f6SMat Martineau mptcp_crypto_key_sha(subflow->remote_key, NULL, &ack_seq); 10776d0060f6SMat Martineau ack_seq++; 1078648ef4b8SMat Martineau subflow->map_seq = ack_seq; 1079648ef4b8SMat Martineau subflow->map_subflow_seq = 1; 10806d0060f6SMat Martineau subflow->rel_write_seq = 1; 10816d0060f6SMat Martineau 1082cec37a6eSPeter Krystad /* the socket is not connected yet, no msk/subflow ops can access/race 1083cec37a6eSPeter Krystad * accessing the field below 1084cec37a6eSPeter Krystad */ 1085cec37a6eSPeter Krystad WRITE_ONCE(msk->remote_key, subflow->remote_key); 1086cec37a6eSPeter Krystad WRITE_ONCE(msk->local_key, subflow->local_key); 108779c0949eSPeter Krystad WRITE_ONCE(msk->token, subflow->token); 10886d0060f6SMat Martineau WRITE_ONCE(msk->write_seq, subflow->idsn + 1); 10896d0060f6SMat Martineau WRITE_ONCE(msk->ack_seq, ack_seq); 1090d22f4988SChristoph Paasch WRITE_ONCE(msk->can_ack, 1); 10911b1c7a0eSPeter Krystad 10921b1c7a0eSPeter Krystad mptcp_pm_new_connection(msk, 0); 1093f870fa0bSMat Martineau } 1094f870fa0bSMat Martineau 1095cf7da0d6SPeter Krystad static void mptcp_sock_graft(struct sock *sk, struct socket *parent) 1096cf7da0d6SPeter Krystad { 1097cf7da0d6SPeter Krystad write_lock_bh(&sk->sk_callback_lock); 1098cf7da0d6SPeter Krystad rcu_assign_pointer(sk->sk_wq, &parent->wq); 1099cf7da0d6SPeter Krystad sk_set_socket(sk, parent); 1100cf7da0d6SPeter Krystad sk->sk_uid = SOCK_INODE(parent)->i_uid; 1101cf7da0d6SPeter Krystad write_unlock_bh(&sk->sk_callback_lock); 1102cf7da0d6SPeter Krystad } 1103cf7da0d6SPeter Krystad 1104*f296234cSPeter Krystad bool mptcp_finish_join(struct sock *sk) 1105*f296234cSPeter Krystad { 1106*f296234cSPeter Krystad struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(sk); 1107*f296234cSPeter Krystad struct mptcp_sock *msk = mptcp_sk(subflow->conn); 1108*f296234cSPeter Krystad struct sock *parent = (void *)msk; 1109*f296234cSPeter Krystad struct socket *parent_sock; 1110*f296234cSPeter Krystad 1111*f296234cSPeter Krystad pr_debug("msk=%p, subflow=%p", msk, subflow); 1112*f296234cSPeter Krystad 1113*f296234cSPeter Krystad /* mptcp socket already closing? */ 1114*f296234cSPeter Krystad if (inet_sk_state_load(parent) != TCP_ESTABLISHED) 1115*f296234cSPeter Krystad return false; 1116*f296234cSPeter Krystad 1117*f296234cSPeter Krystad if (!msk->pm.server_side) 1118*f296234cSPeter Krystad return true; 1119*f296234cSPeter Krystad 1120*f296234cSPeter Krystad /* passive connection, attach to msk socket */ 1121*f296234cSPeter Krystad parent_sock = READ_ONCE(parent->sk_socket); 1122*f296234cSPeter Krystad if (parent_sock && !sk->sk_socket) 1123*f296234cSPeter Krystad mptcp_sock_graft(sk, parent_sock); 1124*f296234cSPeter Krystad 1125*f296234cSPeter Krystad return mptcp_pm_allow_new_subflow(msk); 1126*f296234cSPeter Krystad } 1127*f296234cSPeter Krystad 1128*f296234cSPeter Krystad bool mptcp_sk_is_subflow(const struct sock *sk) 1129*f296234cSPeter Krystad { 1130*f296234cSPeter Krystad struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(sk); 1131*f296234cSPeter Krystad 1132*f296234cSPeter Krystad return subflow->mp_join == 1; 1133*f296234cSPeter Krystad } 1134*f296234cSPeter Krystad 11351891c4a0SFlorian Westphal static bool mptcp_memory_free(const struct sock *sk, int wake) 11361891c4a0SFlorian Westphal { 11371891c4a0SFlorian Westphal struct mptcp_sock *msk = mptcp_sk(sk); 11381891c4a0SFlorian Westphal 11391891c4a0SFlorian Westphal return wake ? test_bit(MPTCP_SEND_SPACE, &msk->flags) : true; 11401891c4a0SFlorian Westphal } 11411891c4a0SFlorian Westphal 1142f870fa0bSMat Martineau static struct proto mptcp_prot = { 1143f870fa0bSMat Martineau .name = "MPTCP", 1144f870fa0bSMat Martineau .owner = THIS_MODULE, 1145f870fa0bSMat Martineau .init = mptcp_init_sock, 1146f870fa0bSMat Martineau .close = mptcp_close, 1147cf7da0d6SPeter Krystad .accept = mptcp_accept, 1148717e79c8SPeter Krystad .setsockopt = mptcp_setsockopt, 1149717e79c8SPeter Krystad .getsockopt = mptcp_getsockopt, 1150f870fa0bSMat Martineau .shutdown = tcp_shutdown, 115179c0949eSPeter Krystad .destroy = mptcp_destroy, 1152f870fa0bSMat Martineau .sendmsg = mptcp_sendmsg, 1153f870fa0bSMat Martineau .recvmsg = mptcp_recvmsg, 115414c441b5SPaolo Abeni .release_cb = mptcp_release_cb, 1155f870fa0bSMat Martineau .hash = inet_hash, 1156f870fa0bSMat Martineau .unhash = inet_unhash, 1157cec37a6eSPeter Krystad .get_port = mptcp_get_port, 11581891c4a0SFlorian Westphal .stream_memory_free = mptcp_memory_free, 1159f870fa0bSMat Martineau .obj_size = sizeof(struct mptcp_sock), 1160f870fa0bSMat Martineau .no_autobind = true, 1161f870fa0bSMat Martineau }; 1162f870fa0bSMat Martineau 11632303f994SPeter Krystad static int mptcp_bind(struct socket *sock, struct sockaddr *uaddr, int addr_len) 11642303f994SPeter Krystad { 11652303f994SPeter Krystad struct mptcp_sock *msk = mptcp_sk(sock->sk); 11662303f994SPeter Krystad struct socket *ssock; 1167cf7da0d6SPeter Krystad int err; 11682303f994SPeter Krystad 11692303f994SPeter Krystad lock_sock(sock->sk); 11702303f994SPeter Krystad ssock = __mptcp_socket_create(msk, MPTCP_SAME_STATE); 11712303f994SPeter Krystad if (IS_ERR(ssock)) { 11722303f994SPeter Krystad err = PTR_ERR(ssock); 11732303f994SPeter Krystad goto unlock; 11742303f994SPeter Krystad } 11752303f994SPeter Krystad 11762303f994SPeter Krystad err = ssock->ops->bind(ssock, uaddr, addr_len); 1177cf7da0d6SPeter Krystad if (!err) 1178cf7da0d6SPeter Krystad mptcp_copy_inaddrs(sock->sk, ssock->sk); 11792303f994SPeter Krystad 11802303f994SPeter Krystad unlock: 11812303f994SPeter Krystad release_sock(sock->sk); 11822303f994SPeter Krystad return err; 11832303f994SPeter Krystad } 11842303f994SPeter Krystad 11852303f994SPeter Krystad static int mptcp_stream_connect(struct socket *sock, struct sockaddr *uaddr, 11862303f994SPeter Krystad int addr_len, int flags) 11872303f994SPeter Krystad { 11882303f994SPeter Krystad struct mptcp_sock *msk = mptcp_sk(sock->sk); 11892303f994SPeter Krystad struct socket *ssock; 11902303f994SPeter Krystad int err; 11912303f994SPeter Krystad 11922303f994SPeter Krystad lock_sock(sock->sk); 11932303f994SPeter Krystad ssock = __mptcp_socket_create(msk, TCP_SYN_SENT); 11942303f994SPeter Krystad if (IS_ERR(ssock)) { 11952303f994SPeter Krystad err = PTR_ERR(ssock); 11962303f994SPeter Krystad goto unlock; 11972303f994SPeter Krystad } 11982303f994SPeter Krystad 1199cf7da0d6SPeter Krystad #ifdef CONFIG_TCP_MD5SIG 1200cf7da0d6SPeter Krystad /* no MPTCP if MD5SIG is enabled on this socket or we may run out of 1201cf7da0d6SPeter Krystad * TCP option space. 1202cf7da0d6SPeter Krystad */ 1203cf7da0d6SPeter Krystad if (rcu_access_pointer(tcp_sk(ssock->sk)->md5sig_info)) 1204cf7da0d6SPeter Krystad mptcp_subflow_ctx(ssock->sk)->request_mptcp = 0; 1205cf7da0d6SPeter Krystad #endif 1206cf7da0d6SPeter Krystad 12072303f994SPeter Krystad err = ssock->ops->connect(ssock, uaddr, addr_len, flags); 12082303f994SPeter Krystad inet_sk_state_store(sock->sk, inet_sk_state_load(ssock->sk)); 1209cf7da0d6SPeter Krystad mptcp_copy_inaddrs(sock->sk, ssock->sk); 12102303f994SPeter Krystad 12112303f994SPeter Krystad unlock: 12122303f994SPeter Krystad release_sock(sock->sk); 12132303f994SPeter Krystad return err; 12142303f994SPeter Krystad } 12152303f994SPeter Krystad 1216cf7da0d6SPeter Krystad static int mptcp_v4_getname(struct socket *sock, struct sockaddr *uaddr, 1217cf7da0d6SPeter Krystad int peer) 1218cf7da0d6SPeter Krystad { 1219cf7da0d6SPeter Krystad if (sock->sk->sk_prot == &tcp_prot) { 1220cf7da0d6SPeter Krystad /* we are being invoked from __sys_accept4, after 1221cf7da0d6SPeter Krystad * mptcp_accept() has just accepted a non-mp-capable 1222cf7da0d6SPeter Krystad * flow: sk is a tcp_sk, not an mptcp one. 1223cf7da0d6SPeter Krystad * 1224cf7da0d6SPeter Krystad * Hand the socket over to tcp so all further socket ops 1225cf7da0d6SPeter Krystad * bypass mptcp. 1226cf7da0d6SPeter Krystad */ 1227cf7da0d6SPeter Krystad sock->ops = &inet_stream_ops; 1228cf7da0d6SPeter Krystad } 1229cf7da0d6SPeter Krystad 1230cf7da0d6SPeter Krystad return inet_getname(sock, uaddr, peer); 1231cf7da0d6SPeter Krystad } 1232cf7da0d6SPeter Krystad 1233cf7da0d6SPeter Krystad #if IS_ENABLED(CONFIG_MPTCP_IPV6) 1234cf7da0d6SPeter Krystad static int mptcp_v6_getname(struct socket *sock, struct sockaddr *uaddr, 1235cf7da0d6SPeter Krystad int peer) 1236cf7da0d6SPeter Krystad { 1237cf7da0d6SPeter Krystad if (sock->sk->sk_prot == &tcpv6_prot) { 1238cf7da0d6SPeter Krystad /* we are being invoked from __sys_accept4 after 1239cf7da0d6SPeter Krystad * mptcp_accept() has accepted a non-mp-capable 1240cf7da0d6SPeter Krystad * subflow: sk is a tcp_sk, not mptcp. 1241cf7da0d6SPeter Krystad * 1242cf7da0d6SPeter Krystad * Hand the socket over to tcp so all further 1243cf7da0d6SPeter Krystad * socket ops bypass mptcp. 1244cf7da0d6SPeter Krystad */ 1245cf7da0d6SPeter Krystad sock->ops = &inet6_stream_ops; 1246cf7da0d6SPeter Krystad } 1247cf7da0d6SPeter Krystad 1248cf7da0d6SPeter Krystad return inet6_getname(sock, uaddr, peer); 1249cf7da0d6SPeter Krystad } 1250cf7da0d6SPeter Krystad #endif 1251cf7da0d6SPeter Krystad 1252cf7da0d6SPeter Krystad static int mptcp_listen(struct socket *sock, int backlog) 1253cf7da0d6SPeter Krystad { 1254cf7da0d6SPeter Krystad struct mptcp_sock *msk = mptcp_sk(sock->sk); 1255cf7da0d6SPeter Krystad struct socket *ssock; 1256cf7da0d6SPeter Krystad int err; 1257cf7da0d6SPeter Krystad 1258cf7da0d6SPeter Krystad pr_debug("msk=%p", msk); 1259cf7da0d6SPeter Krystad 1260cf7da0d6SPeter Krystad lock_sock(sock->sk); 1261cf7da0d6SPeter Krystad ssock = __mptcp_socket_create(msk, TCP_LISTEN); 1262cf7da0d6SPeter Krystad if (IS_ERR(ssock)) { 1263cf7da0d6SPeter Krystad err = PTR_ERR(ssock); 1264cf7da0d6SPeter Krystad goto unlock; 1265cf7da0d6SPeter Krystad } 1266cf7da0d6SPeter Krystad 1267cf7da0d6SPeter Krystad err = ssock->ops->listen(ssock, backlog); 1268cf7da0d6SPeter Krystad inet_sk_state_store(sock->sk, inet_sk_state_load(ssock->sk)); 1269cf7da0d6SPeter Krystad if (!err) 1270cf7da0d6SPeter Krystad mptcp_copy_inaddrs(sock->sk, ssock->sk); 1271cf7da0d6SPeter Krystad 1272cf7da0d6SPeter Krystad unlock: 1273cf7da0d6SPeter Krystad release_sock(sock->sk); 1274cf7da0d6SPeter Krystad return err; 1275cf7da0d6SPeter Krystad } 1276cf7da0d6SPeter Krystad 1277cf7da0d6SPeter Krystad static bool is_tcp_proto(const struct proto *p) 1278cf7da0d6SPeter Krystad { 1279cf7da0d6SPeter Krystad #if IS_ENABLED(CONFIG_MPTCP_IPV6) 1280cf7da0d6SPeter Krystad return p == &tcp_prot || p == &tcpv6_prot; 1281cf7da0d6SPeter Krystad #else 1282cf7da0d6SPeter Krystad return p == &tcp_prot; 1283cf7da0d6SPeter Krystad #endif 1284cf7da0d6SPeter Krystad } 1285cf7da0d6SPeter Krystad 1286cf7da0d6SPeter Krystad static int mptcp_stream_accept(struct socket *sock, struct socket *newsock, 1287cf7da0d6SPeter Krystad int flags, bool kern) 1288cf7da0d6SPeter Krystad { 1289cf7da0d6SPeter Krystad struct mptcp_sock *msk = mptcp_sk(sock->sk); 1290cf7da0d6SPeter Krystad struct socket *ssock; 1291cf7da0d6SPeter Krystad int err; 1292cf7da0d6SPeter Krystad 1293cf7da0d6SPeter Krystad pr_debug("msk=%p", msk); 1294cf7da0d6SPeter Krystad 1295cf7da0d6SPeter Krystad lock_sock(sock->sk); 1296cf7da0d6SPeter Krystad if (sock->sk->sk_state != TCP_LISTEN) 1297cf7da0d6SPeter Krystad goto unlock_fail; 1298cf7da0d6SPeter Krystad 1299cf7da0d6SPeter Krystad ssock = __mptcp_nmpc_socket(msk); 1300cf7da0d6SPeter Krystad if (!ssock) 1301cf7da0d6SPeter Krystad goto unlock_fail; 1302cf7da0d6SPeter Krystad 1303cf7da0d6SPeter Krystad sock_hold(ssock->sk); 1304cf7da0d6SPeter Krystad release_sock(sock->sk); 1305cf7da0d6SPeter Krystad 1306cf7da0d6SPeter Krystad err = ssock->ops->accept(sock, newsock, flags, kern); 1307cf7da0d6SPeter Krystad if (err == 0 && !is_tcp_proto(newsock->sk->sk_prot)) { 1308cf7da0d6SPeter Krystad struct mptcp_sock *msk = mptcp_sk(newsock->sk); 1309cf7da0d6SPeter Krystad struct mptcp_subflow_context *subflow; 1310cf7da0d6SPeter Krystad 1311cf7da0d6SPeter Krystad /* set ssk->sk_socket of accept()ed flows to mptcp socket. 1312cf7da0d6SPeter Krystad * This is needed so NOSPACE flag can be set from tcp stack. 1313cf7da0d6SPeter Krystad */ 1314cf7da0d6SPeter Krystad list_for_each_entry(subflow, &msk->conn_list, node) { 1315cf7da0d6SPeter Krystad struct sock *ssk = mptcp_subflow_tcp_sock(subflow); 1316cf7da0d6SPeter Krystad 1317cf7da0d6SPeter Krystad if (!ssk->sk_socket) 1318cf7da0d6SPeter Krystad mptcp_sock_graft(ssk, newsock); 1319cf7da0d6SPeter Krystad } 1320cf7da0d6SPeter Krystad } 1321cf7da0d6SPeter Krystad 1322cf7da0d6SPeter Krystad sock_put(ssock->sk); 1323cf7da0d6SPeter Krystad return err; 1324cf7da0d6SPeter Krystad 1325cf7da0d6SPeter Krystad unlock_fail: 1326cf7da0d6SPeter Krystad release_sock(sock->sk); 1327cf7da0d6SPeter Krystad return -EINVAL; 1328cf7da0d6SPeter Krystad } 1329cf7da0d6SPeter Krystad 13302303f994SPeter Krystad static __poll_t mptcp_poll(struct file *file, struct socket *sock, 13312303f994SPeter Krystad struct poll_table_struct *wait) 13322303f994SPeter Krystad { 13331891c4a0SFlorian Westphal struct sock *sk = sock->sk; 13348ab183deSPaolo Abeni struct mptcp_sock *msk; 13351891c4a0SFlorian Westphal struct socket *ssock; 13362303f994SPeter Krystad __poll_t mask = 0; 13372303f994SPeter Krystad 13381891c4a0SFlorian Westphal msk = mptcp_sk(sk); 13391891c4a0SFlorian Westphal lock_sock(sk); 13401891c4a0SFlorian Westphal ssock = __mptcp_nmpc_socket(msk); 13411891c4a0SFlorian Westphal if (ssock) { 13421891c4a0SFlorian Westphal mask = ssock->ops->poll(file, ssock, wait); 13431891c4a0SFlorian Westphal release_sock(sk); 13441891c4a0SFlorian Westphal return mask; 13451891c4a0SFlorian Westphal } 13461891c4a0SFlorian Westphal 13471891c4a0SFlorian Westphal release_sock(sk); 13481891c4a0SFlorian Westphal sock_poll_wait(file, sock, wait); 13491891c4a0SFlorian Westphal lock_sock(sk); 13508ab183deSPaolo Abeni ssock = __mptcp_tcp_fallback(msk); 13518ab183deSPaolo Abeni if (unlikely(ssock)) 13528ab183deSPaolo Abeni return ssock->ops->poll(file, ssock, NULL); 13531891c4a0SFlorian Westphal 13541891c4a0SFlorian Westphal if (test_bit(MPTCP_DATA_READY, &msk->flags)) 13551891c4a0SFlorian Westphal mask = EPOLLIN | EPOLLRDNORM; 13561891c4a0SFlorian Westphal if (sk_stream_is_writeable(sk) && 13571891c4a0SFlorian Westphal test_bit(MPTCP_SEND_SPACE, &msk->flags)) 13581891c4a0SFlorian Westphal mask |= EPOLLOUT | EPOLLWRNORM; 13591891c4a0SFlorian Westphal if (sk->sk_shutdown & RCV_SHUTDOWN) 13601891c4a0SFlorian Westphal mask |= EPOLLIN | EPOLLRDNORM | EPOLLRDHUP; 13611891c4a0SFlorian Westphal 13621891c4a0SFlorian Westphal release_sock(sk); 13631891c4a0SFlorian Westphal 13642303f994SPeter Krystad return mask; 13652303f994SPeter Krystad } 13662303f994SPeter Krystad 136721498490SPeter Krystad static int mptcp_shutdown(struct socket *sock, int how) 136821498490SPeter Krystad { 136921498490SPeter Krystad struct mptcp_sock *msk = mptcp_sk(sock->sk); 137021498490SPeter Krystad struct mptcp_subflow_context *subflow; 137121498490SPeter Krystad int ret = 0; 137221498490SPeter Krystad 137321498490SPeter Krystad pr_debug("sk=%p, how=%d", msk, how); 137421498490SPeter Krystad 137521498490SPeter Krystad lock_sock(sock->sk); 137621498490SPeter Krystad 137721498490SPeter Krystad if (how == SHUT_WR || how == SHUT_RDWR) 137821498490SPeter Krystad inet_sk_state_store(sock->sk, TCP_FIN_WAIT1); 137921498490SPeter Krystad 138021498490SPeter Krystad how++; 138121498490SPeter Krystad 138221498490SPeter Krystad if ((how & ~SHUTDOWN_MASK) || !how) { 138321498490SPeter Krystad ret = -EINVAL; 138421498490SPeter Krystad goto out_unlock; 138521498490SPeter Krystad } 138621498490SPeter Krystad 138721498490SPeter Krystad if (sock->state == SS_CONNECTING) { 138821498490SPeter Krystad if ((1 << sock->sk->sk_state) & 138921498490SPeter Krystad (TCPF_SYN_SENT | TCPF_SYN_RECV | TCPF_CLOSE)) 139021498490SPeter Krystad sock->state = SS_DISCONNECTING; 139121498490SPeter Krystad else 139221498490SPeter Krystad sock->state = SS_CONNECTED; 139321498490SPeter Krystad } 139421498490SPeter Krystad 139521498490SPeter Krystad mptcp_for_each_subflow(msk, subflow) { 139621498490SPeter Krystad struct sock *tcp_sk = mptcp_subflow_tcp_sock(subflow); 139721498490SPeter Krystad 139876c42a29SMat Martineau mptcp_subflow_shutdown(tcp_sk, how, 1, msk->write_seq); 139921498490SPeter Krystad } 140021498490SPeter Krystad 140121498490SPeter Krystad out_unlock: 140221498490SPeter Krystad release_sock(sock->sk); 140321498490SPeter Krystad 140421498490SPeter Krystad return ret; 140521498490SPeter Krystad } 140621498490SPeter Krystad 1407e42f1ac6SFlorian Westphal static const struct proto_ops mptcp_stream_ops = { 1408e42f1ac6SFlorian Westphal .family = PF_INET, 1409e42f1ac6SFlorian Westphal .owner = THIS_MODULE, 1410e42f1ac6SFlorian Westphal .release = inet_release, 1411e42f1ac6SFlorian Westphal .bind = mptcp_bind, 1412e42f1ac6SFlorian Westphal .connect = mptcp_stream_connect, 1413e42f1ac6SFlorian Westphal .socketpair = sock_no_socketpair, 1414e42f1ac6SFlorian Westphal .accept = mptcp_stream_accept, 1415e42f1ac6SFlorian Westphal .getname = mptcp_v4_getname, 1416e42f1ac6SFlorian Westphal .poll = mptcp_poll, 1417e42f1ac6SFlorian Westphal .ioctl = inet_ioctl, 1418e42f1ac6SFlorian Westphal .gettstamp = sock_gettstamp, 1419e42f1ac6SFlorian Westphal .listen = mptcp_listen, 1420e42f1ac6SFlorian Westphal .shutdown = mptcp_shutdown, 1421e42f1ac6SFlorian Westphal .setsockopt = sock_common_setsockopt, 1422e42f1ac6SFlorian Westphal .getsockopt = sock_common_getsockopt, 1423e42f1ac6SFlorian Westphal .sendmsg = inet_sendmsg, 1424e42f1ac6SFlorian Westphal .recvmsg = inet_recvmsg, 1425e42f1ac6SFlorian Westphal .mmap = sock_no_mmap, 1426e42f1ac6SFlorian Westphal .sendpage = inet_sendpage, 1427e42f1ac6SFlorian Westphal #ifdef CONFIG_COMPAT 1428e42f1ac6SFlorian Westphal .compat_setsockopt = compat_sock_common_setsockopt, 1429e42f1ac6SFlorian Westphal .compat_getsockopt = compat_sock_common_getsockopt, 1430e42f1ac6SFlorian Westphal #endif 1431e42f1ac6SFlorian Westphal }; 14322303f994SPeter Krystad 1433f870fa0bSMat Martineau static struct inet_protosw mptcp_protosw = { 1434f870fa0bSMat Martineau .type = SOCK_STREAM, 1435f870fa0bSMat Martineau .protocol = IPPROTO_MPTCP, 1436f870fa0bSMat Martineau .prot = &mptcp_prot, 14372303f994SPeter Krystad .ops = &mptcp_stream_ops, 14382303f994SPeter Krystad .flags = INET_PROTOSW_ICSK, 1439f870fa0bSMat Martineau }; 1440f870fa0bSMat Martineau 1441784325e9SMatthieu Baerts void mptcp_proto_init(void) 1442f870fa0bSMat Martineau { 14432303f994SPeter Krystad mptcp_prot.h.hashinfo = tcp_prot.h.hashinfo; 14442303f994SPeter Krystad 14452303f994SPeter Krystad mptcp_subflow_init(); 14461b1c7a0eSPeter Krystad mptcp_pm_init(); 14472303f994SPeter Krystad 1448f870fa0bSMat Martineau if (proto_register(&mptcp_prot, 1) != 0) 1449f870fa0bSMat Martineau panic("Failed to register MPTCP proto.\n"); 1450f870fa0bSMat Martineau 1451f870fa0bSMat Martineau inet_register_protosw(&mptcp_protosw); 14526771bfd9SFlorian Westphal 14536771bfd9SFlorian Westphal BUILD_BUG_ON(sizeof(struct mptcp_skb_cb) > sizeof_field(struct sk_buff, cb)); 1454f870fa0bSMat Martineau } 1455f870fa0bSMat Martineau 1456f870fa0bSMat Martineau #if IS_ENABLED(CONFIG_MPTCP_IPV6) 1457e42f1ac6SFlorian Westphal static const struct proto_ops mptcp_v6_stream_ops = { 1458e42f1ac6SFlorian Westphal .family = PF_INET6, 1459e42f1ac6SFlorian Westphal .owner = THIS_MODULE, 1460e42f1ac6SFlorian Westphal .release = inet6_release, 1461e42f1ac6SFlorian Westphal .bind = mptcp_bind, 1462e42f1ac6SFlorian Westphal .connect = mptcp_stream_connect, 1463e42f1ac6SFlorian Westphal .socketpair = sock_no_socketpair, 1464e42f1ac6SFlorian Westphal .accept = mptcp_stream_accept, 1465e42f1ac6SFlorian Westphal .getname = mptcp_v6_getname, 1466e42f1ac6SFlorian Westphal .poll = mptcp_poll, 1467e42f1ac6SFlorian Westphal .ioctl = inet6_ioctl, 1468e42f1ac6SFlorian Westphal .gettstamp = sock_gettstamp, 1469e42f1ac6SFlorian Westphal .listen = mptcp_listen, 1470e42f1ac6SFlorian Westphal .shutdown = mptcp_shutdown, 1471e42f1ac6SFlorian Westphal .setsockopt = sock_common_setsockopt, 1472e42f1ac6SFlorian Westphal .getsockopt = sock_common_getsockopt, 1473e42f1ac6SFlorian Westphal .sendmsg = inet6_sendmsg, 1474e42f1ac6SFlorian Westphal .recvmsg = inet6_recvmsg, 1475e42f1ac6SFlorian Westphal .mmap = sock_no_mmap, 1476e42f1ac6SFlorian Westphal .sendpage = inet_sendpage, 1477e42f1ac6SFlorian Westphal #ifdef CONFIG_COMPAT 1478e42f1ac6SFlorian Westphal .compat_setsockopt = compat_sock_common_setsockopt, 1479e42f1ac6SFlorian Westphal .compat_getsockopt = compat_sock_common_getsockopt, 1480e42f1ac6SFlorian Westphal #endif 1481e42f1ac6SFlorian Westphal }; 1482e42f1ac6SFlorian Westphal 1483f870fa0bSMat Martineau static struct proto mptcp_v6_prot; 1484f870fa0bSMat Martineau 148579c0949eSPeter Krystad static void mptcp_v6_destroy(struct sock *sk) 148679c0949eSPeter Krystad { 148779c0949eSPeter Krystad mptcp_destroy(sk); 148879c0949eSPeter Krystad inet6_destroy_sock(sk); 148979c0949eSPeter Krystad } 149079c0949eSPeter Krystad 1491f870fa0bSMat Martineau static struct inet_protosw mptcp_v6_protosw = { 1492f870fa0bSMat Martineau .type = SOCK_STREAM, 1493f870fa0bSMat Martineau .protocol = IPPROTO_MPTCP, 1494f870fa0bSMat Martineau .prot = &mptcp_v6_prot, 14952303f994SPeter Krystad .ops = &mptcp_v6_stream_ops, 1496f870fa0bSMat Martineau .flags = INET_PROTOSW_ICSK, 1497f870fa0bSMat Martineau }; 1498f870fa0bSMat Martineau 1499784325e9SMatthieu Baerts int mptcp_proto_v6_init(void) 1500f870fa0bSMat Martineau { 1501f870fa0bSMat Martineau int err; 1502f870fa0bSMat Martineau 1503f870fa0bSMat Martineau mptcp_v6_prot = mptcp_prot; 1504f870fa0bSMat Martineau strcpy(mptcp_v6_prot.name, "MPTCPv6"); 1505f870fa0bSMat Martineau mptcp_v6_prot.slab = NULL; 150679c0949eSPeter Krystad mptcp_v6_prot.destroy = mptcp_v6_destroy; 1507b0519de8SFlorian Westphal mptcp_v6_prot.obj_size = sizeof(struct mptcp6_sock); 1508f870fa0bSMat Martineau 1509f870fa0bSMat Martineau err = proto_register(&mptcp_v6_prot, 1); 1510f870fa0bSMat Martineau if (err) 1511f870fa0bSMat Martineau return err; 1512f870fa0bSMat Martineau 1513f870fa0bSMat Martineau err = inet6_register_protosw(&mptcp_v6_protosw); 1514f870fa0bSMat Martineau if (err) 1515f870fa0bSMat Martineau proto_unregister(&mptcp_v6_prot); 1516f870fa0bSMat Martineau 1517f870fa0bSMat Martineau return err; 1518f870fa0bSMat Martineau } 1519f870fa0bSMat Martineau #endif 1520