1f870fa0bSMat Martineau // SPDX-License-Identifier: GPL-2.0 2f870fa0bSMat Martineau /* Multipath TCP 3f870fa0bSMat Martineau * 4f870fa0bSMat Martineau * Copyright (c) 2017 - 2019, Intel Corporation. 5f870fa0bSMat Martineau */ 6f870fa0bSMat Martineau 7f870fa0bSMat Martineau #define pr_fmt(fmt) "MPTCP: " fmt 8f870fa0bSMat Martineau 9f870fa0bSMat Martineau #include <linux/kernel.h> 10f870fa0bSMat Martineau #include <linux/module.h> 11f870fa0bSMat Martineau #include <linux/netdevice.h> 127a6a6cbcSPaolo Abeni #include <linux/sched/signal.h> 137a6a6cbcSPaolo Abeni #include <linux/atomic.h> 14f870fa0bSMat Martineau #include <net/sock.h> 15f870fa0bSMat Martineau #include <net/inet_common.h> 16f870fa0bSMat Martineau #include <net/inet_hashtables.h> 17f870fa0bSMat Martineau #include <net/protocol.h> 18f870fa0bSMat Martineau #include <net/tcp.h> 19cf7da0d6SPeter Krystad #if IS_ENABLED(CONFIG_MPTCP_IPV6) 20cf7da0d6SPeter Krystad #include <net/transp_v6.h> 21cf7da0d6SPeter Krystad #endif 22f870fa0bSMat Martineau #include <net/mptcp.h> 23f870fa0bSMat Martineau #include "protocol.h" 24fc518953SFlorian Westphal #include "mib.h" 25f870fa0bSMat Martineau 262303f994SPeter Krystad #define MPTCP_SAME_STATE TCP_MAX_STATES 272303f994SPeter Krystad 28b0519de8SFlorian Westphal #if IS_ENABLED(CONFIG_MPTCP_IPV6) 29b0519de8SFlorian Westphal struct mptcp6_sock { 30b0519de8SFlorian Westphal struct mptcp_sock msk; 31b0519de8SFlorian Westphal struct ipv6_pinfo np; 32b0519de8SFlorian Westphal }; 33b0519de8SFlorian Westphal #endif 34b0519de8SFlorian Westphal 356771bfd9SFlorian Westphal struct mptcp_skb_cb { 366771bfd9SFlorian Westphal u32 offset; 376771bfd9SFlorian Westphal }; 386771bfd9SFlorian Westphal 396771bfd9SFlorian Westphal #define MPTCP_SKB_CB(__skb) ((struct mptcp_skb_cb *)&((__skb)->cb[0])) 406771bfd9SFlorian Westphal 41d027236cSPaolo Abeni static struct percpu_counter mptcp_sockets_allocated; 42d027236cSPaolo Abeni 432303f994SPeter Krystad /* If msk has an initial subflow socket, and the MP_CAPABLE handshake has not 442303f994SPeter Krystad * completed yet or has failed, return the subflow socket. 452303f994SPeter Krystad * Otherwise return NULL. 462303f994SPeter Krystad */ 472303f994SPeter Krystad static struct socket *__mptcp_nmpc_socket(const struct mptcp_sock *msk) 482303f994SPeter Krystad { 49d22f4988SChristoph Paasch if (!msk->subflow || READ_ONCE(msk->can_ack)) 502303f994SPeter Krystad return NULL; 512303f994SPeter Krystad 522303f994SPeter Krystad return msk->subflow; 532303f994SPeter Krystad } 542303f994SPeter Krystad 558ab183deSPaolo Abeni static bool __mptcp_needs_tcp_fallback(const struct mptcp_sock *msk) 568ab183deSPaolo Abeni { 578ab183deSPaolo Abeni return msk->first && !sk_is_mptcp(msk->first); 588ab183deSPaolo Abeni } 598ab183deSPaolo Abeni 600b4f33deSFlorian Westphal static struct socket *mptcp_is_tcpsk(struct sock *sk) 610b4f33deSFlorian Westphal { 620b4f33deSFlorian Westphal struct socket *sock = sk->sk_socket; 630b4f33deSFlorian Westphal 640b4f33deSFlorian Westphal if (sock->sk != sk) 650b4f33deSFlorian Westphal return NULL; 660b4f33deSFlorian Westphal 670b4f33deSFlorian Westphal if (unlikely(sk->sk_prot == &tcp_prot)) { 680b4f33deSFlorian Westphal /* we are being invoked after mptcp_accept() has 690b4f33deSFlorian Westphal * accepted a non-mp-capable flow: sk is a tcp_sk, 700b4f33deSFlorian Westphal * not an mptcp one. 710b4f33deSFlorian Westphal * 720b4f33deSFlorian Westphal * Hand the socket over to tcp so all further socket ops 730b4f33deSFlorian Westphal * bypass mptcp. 740b4f33deSFlorian Westphal */ 750b4f33deSFlorian Westphal sock->ops = &inet_stream_ops; 760b4f33deSFlorian Westphal return sock; 770b4f33deSFlorian Westphal #if IS_ENABLED(CONFIG_MPTCP_IPV6) 780b4f33deSFlorian Westphal } else if (unlikely(sk->sk_prot == &tcpv6_prot)) { 790b4f33deSFlorian Westphal sock->ops = &inet6_stream_ops; 800b4f33deSFlorian Westphal return sock; 810b4f33deSFlorian Westphal #endif 820b4f33deSFlorian Westphal } 830b4f33deSFlorian Westphal 840b4f33deSFlorian Westphal return NULL; 850b4f33deSFlorian Westphal } 860b4f33deSFlorian Westphal 878ab183deSPaolo Abeni static struct socket *__mptcp_tcp_fallback(struct mptcp_sock *msk) 88cec37a6eSPeter Krystad { 890b4f33deSFlorian Westphal struct socket *sock; 900b4f33deSFlorian Westphal 91cec37a6eSPeter Krystad sock_owned_by_me((const struct sock *)msk); 92cec37a6eSPeter Krystad 930b4f33deSFlorian Westphal sock = mptcp_is_tcpsk((struct sock *)msk); 940b4f33deSFlorian Westphal if (unlikely(sock)) 950b4f33deSFlorian Westphal return sock; 960b4f33deSFlorian Westphal 978ab183deSPaolo Abeni if (likely(!__mptcp_needs_tcp_fallback(msk))) 98cec37a6eSPeter Krystad return NULL; 99cec37a6eSPeter Krystad 1002c22c06cSFlorian Westphal return msk->subflow; 1018ab183deSPaolo Abeni } 1028ab183deSPaolo Abeni 1032303f994SPeter Krystad static bool __mptcp_can_create_subflow(const struct mptcp_sock *msk) 1042303f994SPeter Krystad { 1058ab183deSPaolo Abeni return !msk->first; 1062303f994SPeter Krystad } 1072303f994SPeter Krystad 1082303f994SPeter Krystad static struct socket *__mptcp_socket_create(struct mptcp_sock *msk, int state) 1092303f994SPeter Krystad { 1102303f994SPeter Krystad struct mptcp_subflow_context *subflow; 1112303f994SPeter Krystad struct sock *sk = (struct sock *)msk; 1122303f994SPeter Krystad struct socket *ssock; 1132303f994SPeter Krystad int err; 1142303f994SPeter Krystad 1150b4f33deSFlorian Westphal ssock = __mptcp_tcp_fallback(msk); 1160b4f33deSFlorian Westphal if (unlikely(ssock)) 1170b4f33deSFlorian Westphal return ssock; 1180b4f33deSFlorian Westphal 1192303f994SPeter Krystad ssock = __mptcp_nmpc_socket(msk); 1202303f994SPeter Krystad if (ssock) 1212303f994SPeter Krystad goto set_state; 1222303f994SPeter Krystad 1232303f994SPeter Krystad if (!__mptcp_can_create_subflow(msk)) 1242303f994SPeter Krystad return ERR_PTR(-EINVAL); 1252303f994SPeter Krystad 1262303f994SPeter Krystad err = mptcp_subflow_create_socket(sk, &ssock); 1272303f994SPeter Krystad if (err) 1282303f994SPeter Krystad return ERR_PTR(err); 1292303f994SPeter Krystad 1308ab183deSPaolo Abeni msk->first = ssock->sk; 1312303f994SPeter Krystad msk->subflow = ssock; 1322303f994SPeter Krystad subflow = mptcp_subflow_ctx(ssock->sk); 133cec37a6eSPeter Krystad list_add(&subflow->node, &msk->conn_list); 1342303f994SPeter Krystad subflow->request_mptcp = 1; 1352303f994SPeter Krystad 1362303f994SPeter Krystad set_state: 1372303f994SPeter Krystad if (state != MPTCP_SAME_STATE) 1382303f994SPeter Krystad inet_sk_state_store(sk, state); 1392303f994SPeter Krystad return ssock; 1402303f994SPeter Krystad } 1412303f994SPeter Krystad 1426771bfd9SFlorian Westphal static void __mptcp_move_skb(struct mptcp_sock *msk, struct sock *ssk, 1436771bfd9SFlorian Westphal struct sk_buff *skb, 1446771bfd9SFlorian Westphal unsigned int offset, size_t copy_len) 1456771bfd9SFlorian Westphal { 1466771bfd9SFlorian Westphal struct sock *sk = (struct sock *)msk; 1476771bfd9SFlorian Westphal 1486771bfd9SFlorian Westphal __skb_unlink(skb, &ssk->sk_receive_queue); 149600911ffSFlorian Westphal skb_set_owner_r(skb, sk); 1506771bfd9SFlorian Westphal __skb_queue_tail(&sk->sk_receive_queue, skb); 1516771bfd9SFlorian Westphal 1526771bfd9SFlorian Westphal msk->ack_seq += copy_len; 1536771bfd9SFlorian Westphal MPTCP_SKB_CB(skb)->offset = offset; 1546771bfd9SFlorian Westphal } 1556771bfd9SFlorian Westphal 156de06f573SFlorian Westphal /* both sockets must be locked */ 157de06f573SFlorian Westphal static bool mptcp_subflow_dsn_valid(const struct mptcp_sock *msk, 158de06f573SFlorian Westphal struct sock *ssk) 159de06f573SFlorian Westphal { 160de06f573SFlorian Westphal struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk); 161de06f573SFlorian Westphal u64 dsn = mptcp_subflow_get_mapped_dsn(subflow); 162de06f573SFlorian Westphal 163de06f573SFlorian Westphal /* revalidate data sequence number. 164de06f573SFlorian Westphal * 165de06f573SFlorian Westphal * mptcp_subflow_data_available() is usually called 166de06f573SFlorian Westphal * without msk lock. Its unlikely (but possible) 167de06f573SFlorian Westphal * that msk->ack_seq has been advanced since the last 168de06f573SFlorian Westphal * call found in-sequence data. 169de06f573SFlorian Westphal */ 170de06f573SFlorian Westphal if (likely(dsn == msk->ack_seq)) 171de06f573SFlorian Westphal return true; 172de06f573SFlorian Westphal 173de06f573SFlorian Westphal subflow->data_avail = 0; 174de06f573SFlorian Westphal return mptcp_subflow_data_available(ssk); 175de06f573SFlorian Westphal } 176de06f573SFlorian Westphal 1776771bfd9SFlorian Westphal static bool __mptcp_move_skbs_from_subflow(struct mptcp_sock *msk, 1786771bfd9SFlorian Westphal struct sock *ssk, 1796771bfd9SFlorian Westphal unsigned int *bytes) 1806771bfd9SFlorian Westphal { 1816771bfd9SFlorian Westphal struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk); 182600911ffSFlorian Westphal struct sock *sk = (struct sock *)msk; 1836771bfd9SFlorian Westphal unsigned int moved = 0; 1846771bfd9SFlorian Westphal bool more_data_avail; 1856771bfd9SFlorian Westphal struct tcp_sock *tp; 1866771bfd9SFlorian Westphal bool done = false; 187600911ffSFlorian Westphal 188de06f573SFlorian Westphal if (!mptcp_subflow_dsn_valid(msk, ssk)) { 189de06f573SFlorian Westphal *bytes = 0; 190de06f573SFlorian Westphal return false; 191de06f573SFlorian Westphal } 192de06f573SFlorian Westphal 193ec33916dSFlorian Westphal if (!(sk->sk_userlocks & SOCK_RCVBUF_LOCK)) { 194ec33916dSFlorian Westphal int rcvbuf = max(ssk->sk_rcvbuf, sk->sk_rcvbuf); 195ec33916dSFlorian Westphal 196600911ffSFlorian Westphal if (rcvbuf > sk->sk_rcvbuf) 197600911ffSFlorian Westphal sk->sk_rcvbuf = rcvbuf; 198ec33916dSFlorian Westphal } 1996771bfd9SFlorian Westphal 2006771bfd9SFlorian Westphal tp = tcp_sk(ssk); 2016771bfd9SFlorian Westphal do { 2026771bfd9SFlorian Westphal u32 map_remaining, offset; 2036771bfd9SFlorian Westphal u32 seq = tp->copied_seq; 2046771bfd9SFlorian Westphal struct sk_buff *skb; 2056771bfd9SFlorian Westphal bool fin; 2066771bfd9SFlorian Westphal 2076771bfd9SFlorian Westphal /* try to move as much data as available */ 2086771bfd9SFlorian Westphal map_remaining = subflow->map_data_len - 2096771bfd9SFlorian Westphal mptcp_subflow_get_map_offset(subflow); 2106771bfd9SFlorian Westphal 2116771bfd9SFlorian Westphal skb = skb_peek(&ssk->sk_receive_queue); 2126771bfd9SFlorian Westphal if (!skb) 2136771bfd9SFlorian Westphal break; 2146771bfd9SFlorian Westphal 2156771bfd9SFlorian Westphal offset = seq - TCP_SKB_CB(skb)->seq; 2166771bfd9SFlorian Westphal fin = TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN; 2176771bfd9SFlorian Westphal if (fin) { 2186771bfd9SFlorian Westphal done = true; 2196771bfd9SFlorian Westphal seq++; 2206771bfd9SFlorian Westphal } 2216771bfd9SFlorian Westphal 2226771bfd9SFlorian Westphal if (offset < skb->len) { 2236771bfd9SFlorian Westphal size_t len = skb->len - offset; 2246771bfd9SFlorian Westphal 2256771bfd9SFlorian Westphal if (tp->urg_data) 2266771bfd9SFlorian Westphal done = true; 2276771bfd9SFlorian Westphal 2286771bfd9SFlorian Westphal __mptcp_move_skb(msk, ssk, skb, offset, len); 2296771bfd9SFlorian Westphal seq += len; 2306771bfd9SFlorian Westphal moved += len; 2316771bfd9SFlorian Westphal 2326771bfd9SFlorian Westphal if (WARN_ON_ONCE(map_remaining < len)) 2336771bfd9SFlorian Westphal break; 2346771bfd9SFlorian Westphal } else { 2356771bfd9SFlorian Westphal WARN_ON_ONCE(!fin); 2366771bfd9SFlorian Westphal sk_eat_skb(ssk, skb); 2376771bfd9SFlorian Westphal done = true; 2386771bfd9SFlorian Westphal } 2396771bfd9SFlorian Westphal 2406771bfd9SFlorian Westphal WRITE_ONCE(tp->copied_seq, seq); 2416771bfd9SFlorian Westphal more_data_avail = mptcp_subflow_data_available(ssk); 242600911ffSFlorian Westphal 243600911ffSFlorian Westphal if (atomic_read(&sk->sk_rmem_alloc) > READ_ONCE(sk->sk_rcvbuf)) { 244600911ffSFlorian Westphal done = true; 245600911ffSFlorian Westphal break; 246600911ffSFlorian Westphal } 2476771bfd9SFlorian Westphal } while (more_data_avail); 2486771bfd9SFlorian Westphal 2496771bfd9SFlorian Westphal *bytes = moved; 2506771bfd9SFlorian Westphal 2516771bfd9SFlorian Westphal return done; 2526771bfd9SFlorian Westphal } 2536771bfd9SFlorian Westphal 2542e52213cSFlorian Westphal /* In most cases we will be able to lock the mptcp socket. If its already 2552e52213cSFlorian Westphal * owned, we need to defer to the work queue to avoid ABBA deadlock. 2562e52213cSFlorian Westphal */ 2572e52213cSFlorian Westphal static bool move_skbs_to_msk(struct mptcp_sock *msk, struct sock *ssk) 2582e52213cSFlorian Westphal { 2592e52213cSFlorian Westphal struct sock *sk = (struct sock *)msk; 2602e52213cSFlorian Westphal unsigned int moved = 0; 2612e52213cSFlorian Westphal 2622e52213cSFlorian Westphal if (READ_ONCE(sk->sk_lock.owned)) 2632e52213cSFlorian Westphal return false; 2642e52213cSFlorian Westphal 2652e52213cSFlorian Westphal if (unlikely(!spin_trylock_bh(&sk->sk_lock.slock))) 2662e52213cSFlorian Westphal return false; 2672e52213cSFlorian Westphal 2682e52213cSFlorian Westphal /* must re-check after taking the lock */ 2692e52213cSFlorian Westphal if (!READ_ONCE(sk->sk_lock.owned)) 2702e52213cSFlorian Westphal __mptcp_move_skbs_from_subflow(msk, ssk, &moved); 2712e52213cSFlorian Westphal 2722e52213cSFlorian Westphal spin_unlock_bh(&sk->sk_lock.slock); 2732e52213cSFlorian Westphal 2742e52213cSFlorian Westphal return moved > 0; 2752e52213cSFlorian Westphal } 2762e52213cSFlorian Westphal 2772e52213cSFlorian Westphal void mptcp_data_ready(struct sock *sk, struct sock *ssk) 278101f6f85SFlorian Westphal { 279101f6f85SFlorian Westphal struct mptcp_sock *msk = mptcp_sk(sk); 280101f6f85SFlorian Westphal 281101f6f85SFlorian Westphal set_bit(MPTCP_DATA_READY, &msk->flags); 2826771bfd9SFlorian Westphal 2832e52213cSFlorian Westphal if (atomic_read(&sk->sk_rmem_alloc) < READ_ONCE(sk->sk_rcvbuf) && 2842e52213cSFlorian Westphal move_skbs_to_msk(msk, ssk)) 2852e52213cSFlorian Westphal goto wake; 2862e52213cSFlorian Westphal 287600911ffSFlorian Westphal /* don't schedule if mptcp sk is (still) over limit */ 288600911ffSFlorian Westphal if (atomic_read(&sk->sk_rmem_alloc) > READ_ONCE(sk->sk_rcvbuf)) 289600911ffSFlorian Westphal goto wake; 290600911ffSFlorian Westphal 29114c441b5SPaolo Abeni /* mptcp socket is owned, release_cb should retry */ 29214c441b5SPaolo Abeni if (!test_and_set_bit(TCP_DELACK_TIMER_DEFERRED, 29314c441b5SPaolo Abeni &sk->sk_tsq_flags)) { 29414c441b5SPaolo Abeni sock_hold(sk); 2956771bfd9SFlorian Westphal 29614c441b5SPaolo Abeni /* need to try again, its possible release_cb() has already 29714c441b5SPaolo Abeni * been called after the test_and_set_bit() above. 29814c441b5SPaolo Abeni */ 29914c441b5SPaolo Abeni move_skbs_to_msk(msk, ssk); 30014c441b5SPaolo Abeni } 301600911ffSFlorian Westphal wake: 302101f6f85SFlorian Westphal sk->sk_data_ready(sk); 303101f6f85SFlorian Westphal } 304101f6f85SFlorian Westphal 305ec3edaa7SPeter Krystad static void __mptcp_flush_join_list(struct mptcp_sock *msk) 306ec3edaa7SPeter Krystad { 307ec3edaa7SPeter Krystad if (likely(list_empty(&msk->join_list))) 308ec3edaa7SPeter Krystad return; 309ec3edaa7SPeter Krystad 310ec3edaa7SPeter Krystad spin_lock_bh(&msk->join_list_lock); 311ec3edaa7SPeter Krystad list_splice_tail_init(&msk->join_list, &msk->conn_list); 312ec3edaa7SPeter Krystad spin_unlock_bh(&msk->join_list_lock); 313ec3edaa7SPeter Krystad } 314ec3edaa7SPeter Krystad 315b51f9b80SPaolo Abeni static void mptcp_set_timeout(const struct sock *sk, const struct sock *ssk) 316b51f9b80SPaolo Abeni { 317b51f9b80SPaolo Abeni long tout = ssk && inet_csk(ssk)->icsk_pending ? 318b51f9b80SPaolo Abeni inet_csk(ssk)->icsk_timeout - jiffies : 0; 319b51f9b80SPaolo Abeni 320b51f9b80SPaolo Abeni if (tout <= 0) 321b51f9b80SPaolo Abeni tout = mptcp_sk(sk)->timer_ival; 322b51f9b80SPaolo Abeni mptcp_sk(sk)->timer_ival = tout > 0 ? tout : TCP_RTO_MIN; 323b51f9b80SPaolo Abeni } 324b51f9b80SPaolo Abeni 325b51f9b80SPaolo Abeni static bool mptcp_timer_pending(struct sock *sk) 326b51f9b80SPaolo Abeni { 327b51f9b80SPaolo Abeni return timer_pending(&inet_csk(sk)->icsk_retransmit_timer); 328b51f9b80SPaolo Abeni } 329b51f9b80SPaolo Abeni 330b51f9b80SPaolo Abeni static void mptcp_reset_timer(struct sock *sk) 331b51f9b80SPaolo Abeni { 332b51f9b80SPaolo Abeni struct inet_connection_sock *icsk = inet_csk(sk); 333b51f9b80SPaolo Abeni unsigned long tout; 334b51f9b80SPaolo Abeni 335b51f9b80SPaolo Abeni /* should never be called with mptcp level timer cleared */ 336b51f9b80SPaolo Abeni tout = READ_ONCE(mptcp_sk(sk)->timer_ival); 337b51f9b80SPaolo Abeni if (WARN_ON_ONCE(!tout)) 338b51f9b80SPaolo Abeni tout = TCP_RTO_MIN; 339b51f9b80SPaolo Abeni sk_reset_timer(sk, &icsk->icsk_retransmit_timer, jiffies + tout); 340b51f9b80SPaolo Abeni } 341b51f9b80SPaolo Abeni 342b51f9b80SPaolo Abeni void mptcp_data_acked(struct sock *sk) 343b51f9b80SPaolo Abeni { 344b51f9b80SPaolo Abeni mptcp_reset_timer(sk); 3453b1d6210SPaolo Abeni 3463b1d6210SPaolo Abeni if (!sk_stream_is_writeable(sk) && 3473b1d6210SPaolo Abeni schedule_work(&mptcp_sk(sk)->work)) 3483b1d6210SPaolo Abeni sock_hold(sk); 349b51f9b80SPaolo Abeni } 350b51f9b80SPaolo Abeni 35159832e24SFlorian Westphal void mptcp_subflow_eof(struct sock *sk) 35259832e24SFlorian Westphal { 35359832e24SFlorian Westphal struct mptcp_sock *msk = mptcp_sk(sk); 35459832e24SFlorian Westphal 35559832e24SFlorian Westphal if (!test_and_set_bit(MPTCP_WORK_EOF, &msk->flags) && 35659832e24SFlorian Westphal schedule_work(&msk->work)) 35759832e24SFlorian Westphal sock_hold(sk); 35859832e24SFlorian Westphal } 35959832e24SFlorian Westphal 360b51f9b80SPaolo Abeni static void mptcp_stop_timer(struct sock *sk) 361b51f9b80SPaolo Abeni { 362b51f9b80SPaolo Abeni struct inet_connection_sock *icsk = inet_csk(sk); 363b51f9b80SPaolo Abeni 364b51f9b80SPaolo Abeni sk_stop_timer(sk, &icsk->icsk_retransmit_timer); 365b51f9b80SPaolo Abeni mptcp_sk(sk)->timer_ival = 0; 366b51f9b80SPaolo Abeni } 367b51f9b80SPaolo Abeni 3686d0060f6SMat Martineau static bool mptcp_ext_cache_refill(struct mptcp_sock *msk) 3696d0060f6SMat Martineau { 3706d0060f6SMat Martineau if (!msk->cached_ext) 3716d0060f6SMat Martineau msk->cached_ext = __skb_ext_alloc(); 3726d0060f6SMat Martineau 3736d0060f6SMat Martineau return !!msk->cached_ext; 3746d0060f6SMat Martineau } 3756d0060f6SMat Martineau 3767a6a6cbcSPaolo Abeni static struct sock *mptcp_subflow_recv_lookup(const struct mptcp_sock *msk) 3777a6a6cbcSPaolo Abeni { 3787a6a6cbcSPaolo Abeni struct mptcp_subflow_context *subflow; 3797a6a6cbcSPaolo Abeni struct sock *sk = (struct sock *)msk; 3807a6a6cbcSPaolo Abeni 3817a6a6cbcSPaolo Abeni sock_owned_by_me(sk); 3827a6a6cbcSPaolo Abeni 3837a6a6cbcSPaolo Abeni mptcp_for_each_subflow(msk, subflow) { 3847a6a6cbcSPaolo Abeni if (subflow->data_avail) 3857a6a6cbcSPaolo Abeni return mptcp_subflow_tcp_sock(subflow); 3867a6a6cbcSPaolo Abeni } 3877a6a6cbcSPaolo Abeni 3887a6a6cbcSPaolo Abeni return NULL; 3897a6a6cbcSPaolo Abeni } 3907a6a6cbcSPaolo Abeni 3913f8e0aaeSPaolo Abeni static bool mptcp_skb_can_collapse_to(u64 write_seq, 39257040755SPaolo Abeni const struct sk_buff *skb, 39357040755SPaolo Abeni const struct mptcp_ext *mpext) 3946d0060f6SMat Martineau { 39557040755SPaolo Abeni if (!tcp_skb_can_collapse_to(skb)) 39657040755SPaolo Abeni return false; 39757040755SPaolo Abeni 39857040755SPaolo Abeni /* can collapse only if MPTCP level sequence is in order */ 3993f8e0aaeSPaolo Abeni return mpext && mpext->data_seq + mpext->data_len == write_seq; 40057040755SPaolo Abeni } 40157040755SPaolo Abeni 40218b683bfSPaolo Abeni static bool mptcp_frag_can_collapse_to(const struct mptcp_sock *msk, 40318b683bfSPaolo Abeni const struct page_frag *pfrag, 40418b683bfSPaolo Abeni const struct mptcp_data_frag *df) 40518b683bfSPaolo Abeni { 40618b683bfSPaolo Abeni return df && pfrag->page == df->page && 40718b683bfSPaolo Abeni df->data_seq + df->data_len == msk->write_seq; 40818b683bfSPaolo Abeni } 40918b683bfSPaolo Abeni 410d027236cSPaolo Abeni static void dfrag_uncharge(struct sock *sk, int len) 41118b683bfSPaolo Abeni { 412d027236cSPaolo Abeni sk_mem_uncharge(sk, len); 4137948f6ccSFlorian Westphal sk_wmem_queued_add(sk, -len); 414d027236cSPaolo Abeni } 415d027236cSPaolo Abeni 416d027236cSPaolo Abeni static void dfrag_clear(struct sock *sk, struct mptcp_data_frag *dfrag) 417d027236cSPaolo Abeni { 418d027236cSPaolo Abeni int len = dfrag->data_len + dfrag->overhead; 419d027236cSPaolo Abeni 42018b683bfSPaolo Abeni list_del(&dfrag->list); 421d027236cSPaolo Abeni dfrag_uncharge(sk, len); 42218b683bfSPaolo Abeni put_page(dfrag->page); 42318b683bfSPaolo Abeni } 42418b683bfSPaolo Abeni 42518b683bfSPaolo Abeni static void mptcp_clean_una(struct sock *sk) 42618b683bfSPaolo Abeni { 42718b683bfSPaolo Abeni struct mptcp_sock *msk = mptcp_sk(sk); 42818b683bfSPaolo Abeni struct mptcp_data_frag *dtmp, *dfrag; 42918b683bfSPaolo Abeni u64 snd_una = atomic64_read(&msk->snd_una); 430d027236cSPaolo Abeni bool cleaned = false; 43118b683bfSPaolo Abeni 43218b683bfSPaolo Abeni list_for_each_entry_safe(dfrag, dtmp, &msk->rtx_queue, list) { 43318b683bfSPaolo Abeni if (after64(dfrag->data_seq + dfrag->data_len, snd_una)) 43418b683bfSPaolo Abeni break; 43518b683bfSPaolo Abeni 436d027236cSPaolo Abeni dfrag_clear(sk, dfrag); 437d027236cSPaolo Abeni cleaned = true; 438d027236cSPaolo Abeni } 439d027236cSPaolo Abeni 4407948f6ccSFlorian Westphal dfrag = mptcp_rtx_head(sk); 4417948f6ccSFlorian Westphal if (dfrag && after64(snd_una, dfrag->data_seq)) { 4427948f6ccSFlorian Westphal u64 delta = dfrag->data_seq + dfrag->data_len - snd_una; 4437948f6ccSFlorian Westphal 4447948f6ccSFlorian Westphal dfrag->data_seq += delta; 4457948f6ccSFlorian Westphal dfrag->data_len -= delta; 4467948f6ccSFlorian Westphal 4477948f6ccSFlorian Westphal dfrag_uncharge(sk, delta); 4487948f6ccSFlorian Westphal cleaned = true; 4497948f6ccSFlorian Westphal } 4507948f6ccSFlorian Westphal 451d027236cSPaolo Abeni if (cleaned) { 452d027236cSPaolo Abeni sk_mem_reclaim_partial(sk); 4537948f6ccSFlorian Westphal 4547948f6ccSFlorian Westphal /* Only wake up writers if a subflow is ready */ 4557948f6ccSFlorian Westphal if (test_bit(MPTCP_SEND_SPACE, &msk->flags)) 4567948f6ccSFlorian Westphal sk_stream_write_space(sk); 45718b683bfSPaolo Abeni } 45818b683bfSPaolo Abeni } 45918b683bfSPaolo Abeni 46018b683bfSPaolo Abeni /* ensure we get enough memory for the frag hdr, beyond some minimal amount of 46118b683bfSPaolo Abeni * data 46218b683bfSPaolo Abeni */ 46318b683bfSPaolo Abeni static bool mptcp_page_frag_refill(struct sock *sk, struct page_frag *pfrag) 46418b683bfSPaolo Abeni { 46518b683bfSPaolo Abeni if (likely(skb_page_frag_refill(32U + sizeof(struct mptcp_data_frag), 46618b683bfSPaolo Abeni pfrag, sk->sk_allocation))) 46718b683bfSPaolo Abeni return true; 46818b683bfSPaolo Abeni 46918b683bfSPaolo Abeni sk->sk_prot->enter_memory_pressure(sk); 47018b683bfSPaolo Abeni sk_stream_moderate_sndbuf(sk); 47118b683bfSPaolo Abeni return false; 47218b683bfSPaolo Abeni } 47318b683bfSPaolo Abeni 47418b683bfSPaolo Abeni static struct mptcp_data_frag * 47518b683bfSPaolo Abeni mptcp_carve_data_frag(const struct mptcp_sock *msk, struct page_frag *pfrag, 47618b683bfSPaolo Abeni int orig_offset) 47718b683bfSPaolo Abeni { 47818b683bfSPaolo Abeni int offset = ALIGN(orig_offset, sizeof(long)); 47918b683bfSPaolo Abeni struct mptcp_data_frag *dfrag; 48018b683bfSPaolo Abeni 48118b683bfSPaolo Abeni dfrag = (struct mptcp_data_frag *)(page_to_virt(pfrag->page) + offset); 48218b683bfSPaolo Abeni dfrag->data_len = 0; 48318b683bfSPaolo Abeni dfrag->data_seq = msk->write_seq; 48418b683bfSPaolo Abeni dfrag->overhead = offset - orig_offset + sizeof(struct mptcp_data_frag); 48518b683bfSPaolo Abeni dfrag->offset = offset + sizeof(struct mptcp_data_frag); 48618b683bfSPaolo Abeni dfrag->page = pfrag->page; 48718b683bfSPaolo Abeni 48818b683bfSPaolo Abeni return dfrag; 48918b683bfSPaolo Abeni } 49018b683bfSPaolo Abeni 49157040755SPaolo Abeni static int mptcp_sendmsg_frag(struct sock *sk, struct sock *ssk, 4923f8e0aaeSPaolo Abeni struct msghdr *msg, struct mptcp_data_frag *dfrag, 4933f8e0aaeSPaolo Abeni long *timeo, int *pmss_now, 49457040755SPaolo Abeni int *ps_goal) 49557040755SPaolo Abeni { 49618b683bfSPaolo Abeni int mss_now, avail_size, size_goal, offset, ret, frag_truesize = 0; 49718b683bfSPaolo Abeni bool dfrag_collapsed, can_collapse = false; 4986d0060f6SMat Martineau struct mptcp_sock *msk = mptcp_sk(sk); 4996d0060f6SMat Martineau struct mptcp_ext *mpext = NULL; 5003f8e0aaeSPaolo Abeni bool retransmission = !!dfrag; 50157040755SPaolo Abeni struct sk_buff *skb, *tail; 5026d0060f6SMat Martineau struct page_frag *pfrag; 5033f8e0aaeSPaolo Abeni struct page *page; 5043f8e0aaeSPaolo Abeni u64 *write_seq; 5056d0060f6SMat Martineau size_t psize; 5066d0060f6SMat Martineau 5076d0060f6SMat Martineau /* use the mptcp page cache so that we can easily move the data 5086d0060f6SMat Martineau * from one substream to another, but do per subflow memory accounting 5093f8e0aaeSPaolo Abeni * Note: pfrag is used only !retransmission, but the compiler if 5103f8e0aaeSPaolo Abeni * fooled into a warning if we don't init here 5116d0060f6SMat Martineau */ 5126d0060f6SMat Martineau pfrag = sk_page_frag(sk); 5133f8e0aaeSPaolo Abeni while ((!retransmission && !mptcp_page_frag_refill(ssk, pfrag)) || 5146d0060f6SMat Martineau !mptcp_ext_cache_refill(msk)) { 5156d0060f6SMat Martineau ret = sk_stream_wait_memory(ssk, timeo); 5166d0060f6SMat Martineau if (ret) 5176d0060f6SMat Martineau return ret; 51818b683bfSPaolo Abeni 51918b683bfSPaolo Abeni /* if sk_stream_wait_memory() sleeps snd_una can change 52018b683bfSPaolo Abeni * significantly, refresh the rtx queue 52118b683bfSPaolo Abeni */ 52218b683bfSPaolo Abeni mptcp_clean_una(sk); 52318b683bfSPaolo Abeni 5248ab183deSPaolo Abeni if (unlikely(__mptcp_needs_tcp_fallback(msk))) 5258ab183deSPaolo Abeni return 0; 5266d0060f6SMat Martineau } 5273f8e0aaeSPaolo Abeni if (!retransmission) { 5283f8e0aaeSPaolo Abeni write_seq = &msk->write_seq; 5293f8e0aaeSPaolo Abeni page = pfrag->page; 5303f8e0aaeSPaolo Abeni } else { 5313f8e0aaeSPaolo Abeni write_seq = &dfrag->data_seq; 5323f8e0aaeSPaolo Abeni page = dfrag->page; 5333f8e0aaeSPaolo Abeni } 5346d0060f6SMat Martineau 5356d0060f6SMat Martineau /* compute copy limit */ 5366d0060f6SMat Martineau mss_now = tcp_send_mss(ssk, &size_goal, msg->msg_flags); 53757040755SPaolo Abeni *pmss_now = mss_now; 53857040755SPaolo Abeni *ps_goal = size_goal; 53957040755SPaolo Abeni avail_size = size_goal; 54057040755SPaolo Abeni skb = tcp_write_queue_tail(ssk); 54157040755SPaolo Abeni if (skb) { 54257040755SPaolo Abeni mpext = skb_ext_find(skb, SKB_EXT_MPTCP); 5436d0060f6SMat Martineau 54457040755SPaolo Abeni /* Limit the write to the size available in the 54557040755SPaolo Abeni * current skb, if any, so that we create at most a new skb. 54657040755SPaolo Abeni * Explicitly tells TCP internals to avoid collapsing on later 54757040755SPaolo Abeni * queue management operation, to avoid breaking the ext <-> 54857040755SPaolo Abeni * SSN association set here 54957040755SPaolo Abeni */ 55057040755SPaolo Abeni can_collapse = (size_goal - skb->len > 0) && 5513f8e0aaeSPaolo Abeni mptcp_skb_can_collapse_to(*write_seq, skb, mpext); 55257040755SPaolo Abeni if (!can_collapse) 55357040755SPaolo Abeni TCP_SKB_CB(skb)->eor = 1; 55457040755SPaolo Abeni else 55557040755SPaolo Abeni avail_size = size_goal - skb->len; 55657040755SPaolo Abeni } 55718b683bfSPaolo Abeni 5583f8e0aaeSPaolo Abeni if (!retransmission) { 5593f8e0aaeSPaolo Abeni /* reuse tail pfrag, if possible, or carve a new one from the 5603f8e0aaeSPaolo Abeni * page allocator 56118b683bfSPaolo Abeni */ 56218b683bfSPaolo Abeni dfrag = mptcp_rtx_tail(sk); 56318b683bfSPaolo Abeni offset = pfrag->offset; 56418b683bfSPaolo Abeni dfrag_collapsed = mptcp_frag_can_collapse_to(msk, pfrag, dfrag); 56518b683bfSPaolo Abeni if (!dfrag_collapsed) { 56618b683bfSPaolo Abeni dfrag = mptcp_carve_data_frag(msk, pfrag, offset); 56718b683bfSPaolo Abeni offset = dfrag->offset; 56818b683bfSPaolo Abeni frag_truesize = dfrag->overhead; 56918b683bfSPaolo Abeni } 57018b683bfSPaolo Abeni psize = min_t(size_t, pfrag->size - offset, avail_size); 57157040755SPaolo Abeni 57257040755SPaolo Abeni /* Copy to page */ 5736d0060f6SMat Martineau pr_debug("left=%zu", msg_data_left(msg)); 57418b683bfSPaolo Abeni psize = copy_page_from_iter(pfrag->page, offset, 5753f8e0aaeSPaolo Abeni min_t(size_t, msg_data_left(msg), 5763f8e0aaeSPaolo Abeni psize), 5776d0060f6SMat Martineau &msg->msg_iter); 5786d0060f6SMat Martineau pr_debug("left=%zu", msg_data_left(msg)); 5796d0060f6SMat Martineau if (!psize) 5806d0060f6SMat Martineau return -EINVAL; 5816d0060f6SMat Martineau 582d027236cSPaolo Abeni if (!sk_wmem_schedule(sk, psize + dfrag->overhead)) 583d027236cSPaolo Abeni return -ENOMEM; 5843f8e0aaeSPaolo Abeni } else { 5853f8e0aaeSPaolo Abeni offset = dfrag->offset; 5863f8e0aaeSPaolo Abeni psize = min_t(size_t, dfrag->data_len, avail_size); 5873f8e0aaeSPaolo Abeni } 588d027236cSPaolo Abeni 58957040755SPaolo Abeni /* tell the TCP stack to delay the push so that we can safely 59057040755SPaolo Abeni * access the skb after the sendpages call 5916d0060f6SMat Martineau */ 5923f8e0aaeSPaolo Abeni ret = do_tcp_sendpages(ssk, page, offset, psize, 5936d0060f6SMat Martineau msg->msg_flags | MSG_SENDPAGE_NOTLAST); 5946d0060f6SMat Martineau if (ret <= 0) 5956d0060f6SMat Martineau return ret; 59618b683bfSPaolo Abeni 59718b683bfSPaolo Abeni frag_truesize += ret; 5983f8e0aaeSPaolo Abeni if (!retransmission) { 5996d0060f6SMat Martineau if (unlikely(ret < psize)) 6006d0060f6SMat Martineau iov_iter_revert(&msg->msg_iter, psize - ret); 6016d0060f6SMat Martineau 60218b683bfSPaolo Abeni /* send successful, keep track of sent data for mptcp-level 60318b683bfSPaolo Abeni * retransmission 60418b683bfSPaolo Abeni */ 60518b683bfSPaolo Abeni dfrag->data_len += ret; 60618b683bfSPaolo Abeni if (!dfrag_collapsed) { 60718b683bfSPaolo Abeni get_page(dfrag->page); 60818b683bfSPaolo Abeni list_add_tail(&dfrag->list, &msk->rtx_queue); 6093f8e0aaeSPaolo Abeni sk_wmem_queued_add(sk, frag_truesize); 6103f8e0aaeSPaolo Abeni } else { 6113f8e0aaeSPaolo Abeni sk_wmem_queued_add(sk, ret); 61218b683bfSPaolo Abeni } 61318b683bfSPaolo Abeni 614d027236cSPaolo Abeni /* charge data on mptcp rtx queue to the master socket 615d027236cSPaolo Abeni * Note: we charge such data both to sk and ssk 616d027236cSPaolo Abeni */ 617d027236cSPaolo Abeni sk->sk_forward_alloc -= frag_truesize; 6183f8e0aaeSPaolo Abeni } 619d027236cSPaolo Abeni 62057040755SPaolo Abeni /* if the tail skb extension is still the cached one, collapsing 62157040755SPaolo Abeni * really happened. Note: we can't check for 'same skb' as the sk_buff 62257040755SPaolo Abeni * hdr on tail can be transmitted, freed and re-allocated by the 62357040755SPaolo Abeni * do_tcp_sendpages() call 62457040755SPaolo Abeni */ 62557040755SPaolo Abeni tail = tcp_write_queue_tail(ssk); 62657040755SPaolo Abeni if (mpext && tail && mpext == skb_ext_find(tail, SKB_EXT_MPTCP)) { 62757040755SPaolo Abeni WARN_ON_ONCE(!can_collapse); 62857040755SPaolo Abeni mpext->data_len += ret; 62957040755SPaolo Abeni goto out; 63057040755SPaolo Abeni } 63157040755SPaolo Abeni 6326d0060f6SMat Martineau skb = tcp_write_queue_tail(ssk); 6336d0060f6SMat Martineau mpext = __skb_ext_set(skb, SKB_EXT_MPTCP, msk->cached_ext); 6346d0060f6SMat Martineau msk->cached_ext = NULL; 6356d0060f6SMat Martineau 6366d0060f6SMat Martineau memset(mpext, 0, sizeof(*mpext)); 6373f8e0aaeSPaolo Abeni mpext->data_seq = *write_seq; 6386d0060f6SMat Martineau mpext->subflow_seq = mptcp_subflow_ctx(ssk)->rel_write_seq; 6396d0060f6SMat Martineau mpext->data_len = ret; 6406d0060f6SMat Martineau mpext->use_map = 1; 6416d0060f6SMat Martineau mpext->dsn64 = 1; 6426d0060f6SMat Martineau 6436d0060f6SMat Martineau pr_debug("data_seq=%llu subflow_seq=%u data_len=%u dsn64=%d", 6446d0060f6SMat Martineau mpext->data_seq, mpext->subflow_seq, mpext->data_len, 6456d0060f6SMat Martineau mpext->dsn64); 6466d0060f6SMat Martineau 64757040755SPaolo Abeni out: 6483f8e0aaeSPaolo Abeni if (!retransmission) 64918b683bfSPaolo Abeni pfrag->offset += frag_truesize; 6503f8e0aaeSPaolo Abeni *write_seq += ret; 6516d0060f6SMat Martineau mptcp_subflow_ctx(ssk)->rel_write_seq += ret; 6526d0060f6SMat Martineau 6536d0060f6SMat Martineau return ret; 6546d0060f6SMat Martineau } 6556d0060f6SMat Martineau 656a0e17064SFlorian Westphal static void mptcp_nospace(struct mptcp_sock *msk, struct socket *sock) 657a0e17064SFlorian Westphal { 658a0e17064SFlorian Westphal clear_bit(MPTCP_SEND_SPACE, &msk->flags); 659a0e17064SFlorian Westphal smp_mb__after_atomic(); /* msk->flags is changed by write_space cb */ 660a0e17064SFlorian Westphal 661a0e17064SFlorian Westphal /* enables sk->write_space() callbacks */ 662a0e17064SFlorian Westphal set_bit(SOCK_NOSPACE, &sock->flags); 663a0e17064SFlorian Westphal } 664a0e17064SFlorian Westphal 665f296234cSPeter Krystad static struct sock *mptcp_subflow_get_send(struct mptcp_sock *msk) 666f296234cSPeter Krystad { 667f296234cSPeter Krystad struct mptcp_subflow_context *subflow; 668f296234cSPeter Krystad struct sock *backup = NULL; 669f296234cSPeter Krystad 670f296234cSPeter Krystad sock_owned_by_me((const struct sock *)msk); 671f296234cSPeter Krystad 672f296234cSPeter Krystad mptcp_for_each_subflow(msk, subflow) { 673f296234cSPeter Krystad struct sock *ssk = mptcp_subflow_tcp_sock(subflow); 674f296234cSPeter Krystad 675f296234cSPeter Krystad if (!sk_stream_memory_free(ssk)) { 676f296234cSPeter Krystad struct socket *sock = ssk->sk_socket; 677f296234cSPeter Krystad 678a0e17064SFlorian Westphal if (sock) 679a0e17064SFlorian Westphal mptcp_nospace(msk, sock); 680f296234cSPeter Krystad 681f296234cSPeter Krystad return NULL; 682f296234cSPeter Krystad } 683f296234cSPeter Krystad 684f296234cSPeter Krystad if (subflow->backup) { 685f296234cSPeter Krystad if (!backup) 686f296234cSPeter Krystad backup = ssk; 687f296234cSPeter Krystad 688f296234cSPeter Krystad continue; 689f296234cSPeter Krystad } 690f296234cSPeter Krystad 691f296234cSPeter Krystad return ssk; 692f296234cSPeter Krystad } 693f296234cSPeter Krystad 694f296234cSPeter Krystad return backup; 695f296234cSPeter Krystad } 696f296234cSPeter Krystad 6971891c4a0SFlorian Westphal static void ssk_check_wmem(struct mptcp_sock *msk, struct sock *ssk) 6981891c4a0SFlorian Westphal { 6991891c4a0SFlorian Westphal struct socket *sock; 7001891c4a0SFlorian Westphal 7011891c4a0SFlorian Westphal if (likely(sk_stream_is_writeable(ssk))) 7021891c4a0SFlorian Westphal return; 7031891c4a0SFlorian Westphal 7041891c4a0SFlorian Westphal sock = READ_ONCE(ssk->sk_socket); 705a0e17064SFlorian Westphal if (sock) 706a0e17064SFlorian Westphal mptcp_nospace(msk, sock); 7071891c4a0SFlorian Westphal } 7081891c4a0SFlorian Westphal 709f870fa0bSMat Martineau static int mptcp_sendmsg(struct sock *sk, struct msghdr *msg, size_t len) 710f870fa0bSMat Martineau { 71157040755SPaolo Abeni int mss_now = 0, size_goal = 0, ret = 0; 712f870fa0bSMat Martineau struct mptcp_sock *msk = mptcp_sk(sk); 713cec37a6eSPeter Krystad struct socket *ssock; 7146d0060f6SMat Martineau size_t copied = 0; 715cec37a6eSPeter Krystad struct sock *ssk; 7166d0060f6SMat Martineau long timeo; 717f870fa0bSMat Martineau 718f870fa0bSMat Martineau if (msg->msg_flags & ~(MSG_MORE | MSG_DONTWAIT | MSG_NOSIGNAL)) 719f870fa0bSMat Martineau return -EOPNOTSUPP; 720f870fa0bSMat Martineau 721cec37a6eSPeter Krystad lock_sock(sk); 7221954b860SMat Martineau 7231954b860SMat Martineau timeo = sock_sndtimeo(sk, msg->msg_flags & MSG_DONTWAIT); 7241954b860SMat Martineau 7251954b860SMat Martineau if ((1 << sk->sk_state) & ~(TCPF_ESTABLISHED | TCPF_CLOSE_WAIT)) { 7261954b860SMat Martineau ret = sk_stream_wait_connect(sk, &timeo); 7271954b860SMat Martineau if (ret) 7281954b860SMat Martineau goto out; 7291954b860SMat Martineau } 7301954b860SMat Martineau 731e154659bSFlorian Westphal fallback: 732cec37a6eSPeter Krystad ssock = __mptcp_tcp_fallback(msk); 7338ab183deSPaolo Abeni if (unlikely(ssock)) { 734e154659bSFlorian Westphal release_sock(sk); 735cec37a6eSPeter Krystad pr_debug("fallback passthrough"); 736cec37a6eSPeter Krystad ret = sock_sendmsg(ssock, msg); 7378ab183deSPaolo Abeni return ret >= 0 ? ret + copied : (copied ? copied : ret); 738cec37a6eSPeter Krystad } 739cec37a6eSPeter Krystad 74018b683bfSPaolo Abeni mptcp_clean_una(sk); 74118b683bfSPaolo Abeni 742*fb529e62SFlorian Westphal wait_for_sndbuf: 743ec3edaa7SPeter Krystad __mptcp_flush_join_list(msk); 744f296234cSPeter Krystad ssk = mptcp_subflow_get_send(msk); 745f296234cSPeter Krystad while (!sk_stream_memory_free(sk) || !ssk) { 746*fb529e62SFlorian Westphal if (ssk) { 747*fb529e62SFlorian Westphal /* make sure retransmit timer is 748*fb529e62SFlorian Westphal * running before we wait for memory. 749*fb529e62SFlorian Westphal * 750*fb529e62SFlorian Westphal * The retransmit timer might be needed 751*fb529e62SFlorian Westphal * to make the peer send an up-to-date 752*fb529e62SFlorian Westphal * MPTCP Ack. 753*fb529e62SFlorian Westphal */ 754*fb529e62SFlorian Westphal mptcp_set_timeout(sk, ssk); 755*fb529e62SFlorian Westphal if (!mptcp_timer_pending(sk)) 756*fb529e62SFlorian Westphal mptcp_reset_timer(sk); 757*fb529e62SFlorian Westphal } 758*fb529e62SFlorian Westphal 759f296234cSPeter Krystad ret = sk_stream_wait_memory(sk, &timeo); 760f296234cSPeter Krystad if (ret) 761f296234cSPeter Krystad goto out; 762f296234cSPeter Krystad 76318b683bfSPaolo Abeni mptcp_clean_una(sk); 76418b683bfSPaolo Abeni 765f296234cSPeter Krystad ssk = mptcp_subflow_get_send(msk); 766f296234cSPeter Krystad if (list_empty(&msk->conn_list)) { 767f296234cSPeter Krystad ret = -ENOTCONN; 768f296234cSPeter Krystad goto out; 769f296234cSPeter Krystad } 770cec37a6eSPeter Krystad } 771cec37a6eSPeter Krystad 7726d0060f6SMat Martineau pr_debug("conn_list->subflow=%p", ssk); 773cec37a6eSPeter Krystad 7746d0060f6SMat Martineau lock_sock(ssk); 7756d0060f6SMat Martineau while (msg_data_left(msg)) { 7763f8e0aaeSPaolo Abeni ret = mptcp_sendmsg_frag(sk, ssk, msg, NULL, &timeo, &mss_now, 77757040755SPaolo Abeni &size_goal); 7786d0060f6SMat Martineau if (ret < 0) 7796d0060f6SMat Martineau break; 7808ab183deSPaolo Abeni if (ret == 0 && unlikely(__mptcp_needs_tcp_fallback(msk))) { 781e154659bSFlorian Westphal /* Can happen for passive sockets: 782e154659bSFlorian Westphal * 3WHS negotiated MPTCP, but first packet after is 783e154659bSFlorian Westphal * plain TCP (e.g. due to middlebox filtering unknown 784e154659bSFlorian Westphal * options). 785e154659bSFlorian Westphal * 786e154659bSFlorian Westphal * Fall back to TCP. 787e154659bSFlorian Westphal */ 7888ab183deSPaolo Abeni release_sock(ssk); 7898ab183deSPaolo Abeni goto fallback; 7908ab183deSPaolo Abeni } 7916d0060f6SMat Martineau 7926d0060f6SMat Martineau copied += ret; 793*fb529e62SFlorian Westphal 794*fb529e62SFlorian Westphal /* memory is charged to mptcp level socket as well, i.e. 795*fb529e62SFlorian Westphal * if msg is very large, mptcp socket may run out of buffer 796*fb529e62SFlorian Westphal * space. mptcp_clean_una() will release data that has 797*fb529e62SFlorian Westphal * been acked at mptcp level in the mean time, so there is 798*fb529e62SFlorian Westphal * a good chance we can continue sending data right away. 799*fb529e62SFlorian Westphal */ 800*fb529e62SFlorian Westphal if (unlikely(!sk_stream_memory_free(sk))) { 801*fb529e62SFlorian Westphal tcp_push(ssk, msg->msg_flags, mss_now, 802*fb529e62SFlorian Westphal tcp_sk(ssk)->nonagle, size_goal); 803*fb529e62SFlorian Westphal mptcp_clean_una(sk); 804*fb529e62SFlorian Westphal if (!sk_stream_memory_free(sk)) { 805*fb529e62SFlorian Westphal /* can't send more for now, need to wait for 806*fb529e62SFlorian Westphal * MPTCP-level ACKs from peer. 807*fb529e62SFlorian Westphal * 808*fb529e62SFlorian Westphal * Wakeup will happen via mptcp_clean_una(). 809*fb529e62SFlorian Westphal */ 810*fb529e62SFlorian Westphal mptcp_set_timeout(sk, ssk); 811*fb529e62SFlorian Westphal release_sock(ssk); 812*fb529e62SFlorian Westphal goto wait_for_sndbuf; 813*fb529e62SFlorian Westphal } 814*fb529e62SFlorian Westphal } 8156d0060f6SMat Martineau } 8166d0060f6SMat Martineau 817b51f9b80SPaolo Abeni mptcp_set_timeout(sk, ssk); 81857040755SPaolo Abeni if (copied) { 8196d0060f6SMat Martineau ret = copied; 82057040755SPaolo Abeni tcp_push(ssk, msg->msg_flags, mss_now, tcp_sk(ssk)->nonagle, 82157040755SPaolo Abeni size_goal); 822b51f9b80SPaolo Abeni 823b51f9b80SPaolo Abeni /* start the timer, if it's not pending */ 824b51f9b80SPaolo Abeni if (!mptcp_timer_pending(sk)) 825b51f9b80SPaolo Abeni mptcp_reset_timer(sk); 82657040755SPaolo Abeni } 8276d0060f6SMat Martineau 8281891c4a0SFlorian Westphal ssk_check_wmem(msk, ssk); 8296d0060f6SMat Martineau release_sock(ssk); 8301954b860SMat Martineau out: 831cec37a6eSPeter Krystad release_sock(sk); 832cec37a6eSPeter Krystad return ret; 833f870fa0bSMat Martineau } 834f870fa0bSMat Martineau 8357a6a6cbcSPaolo Abeni static void mptcp_wait_data(struct sock *sk, long *timeo) 8367a6a6cbcSPaolo Abeni { 8377a6a6cbcSPaolo Abeni DEFINE_WAIT_FUNC(wait, woken_wake_function); 8387a6a6cbcSPaolo Abeni struct mptcp_sock *msk = mptcp_sk(sk); 8397a6a6cbcSPaolo Abeni 8407a6a6cbcSPaolo Abeni add_wait_queue(sk_sleep(sk), &wait); 8417a6a6cbcSPaolo Abeni sk_set_bit(SOCKWQ_ASYNC_WAITDATA, sk); 8427a6a6cbcSPaolo Abeni 8437a6a6cbcSPaolo Abeni sk_wait_event(sk, timeo, 8447a6a6cbcSPaolo Abeni test_and_clear_bit(MPTCP_DATA_READY, &msk->flags), &wait); 8457a6a6cbcSPaolo Abeni 8467a6a6cbcSPaolo Abeni sk_clear_bit(SOCKWQ_ASYNC_WAITDATA, sk); 8477a6a6cbcSPaolo Abeni remove_wait_queue(sk_sleep(sk), &wait); 8487a6a6cbcSPaolo Abeni } 8497a6a6cbcSPaolo Abeni 8506771bfd9SFlorian Westphal static int __mptcp_recvmsg_mskq(struct mptcp_sock *msk, 8516771bfd9SFlorian Westphal struct msghdr *msg, 8526771bfd9SFlorian Westphal size_t len) 8536771bfd9SFlorian Westphal { 8546771bfd9SFlorian Westphal struct sock *sk = (struct sock *)msk; 8556771bfd9SFlorian Westphal struct sk_buff *skb; 8566771bfd9SFlorian Westphal int copied = 0; 8576771bfd9SFlorian Westphal 8586771bfd9SFlorian Westphal while ((skb = skb_peek(&sk->sk_receive_queue)) != NULL) { 8596771bfd9SFlorian Westphal u32 offset = MPTCP_SKB_CB(skb)->offset; 8606771bfd9SFlorian Westphal u32 data_len = skb->len - offset; 8616771bfd9SFlorian Westphal u32 count = min_t(size_t, len - copied, data_len); 8626771bfd9SFlorian Westphal int err; 8636771bfd9SFlorian Westphal 8646771bfd9SFlorian Westphal err = skb_copy_datagram_msg(skb, offset, msg, count); 8656771bfd9SFlorian Westphal if (unlikely(err < 0)) { 8666771bfd9SFlorian Westphal if (!copied) 8676771bfd9SFlorian Westphal return err; 8686771bfd9SFlorian Westphal break; 8696771bfd9SFlorian Westphal } 8706771bfd9SFlorian Westphal 8716771bfd9SFlorian Westphal copied += count; 8726771bfd9SFlorian Westphal 8736771bfd9SFlorian Westphal if (count < data_len) { 8746771bfd9SFlorian Westphal MPTCP_SKB_CB(skb)->offset += count; 8756771bfd9SFlorian Westphal break; 8766771bfd9SFlorian Westphal } 8776771bfd9SFlorian Westphal 8786771bfd9SFlorian Westphal __skb_unlink(skb, &sk->sk_receive_queue); 8796771bfd9SFlorian Westphal __kfree_skb(skb); 8806771bfd9SFlorian Westphal 8816771bfd9SFlorian Westphal if (copied >= len) 8826771bfd9SFlorian Westphal break; 8836771bfd9SFlorian Westphal } 8846771bfd9SFlorian Westphal 8856771bfd9SFlorian Westphal return copied; 8866771bfd9SFlorian Westphal } 8876771bfd9SFlorian Westphal 8886771bfd9SFlorian Westphal static bool __mptcp_move_skbs(struct mptcp_sock *msk) 8896771bfd9SFlorian Westphal { 8906771bfd9SFlorian Westphal unsigned int moved = 0; 8916771bfd9SFlorian Westphal bool done; 8926771bfd9SFlorian Westphal 8936771bfd9SFlorian Westphal do { 8946771bfd9SFlorian Westphal struct sock *ssk = mptcp_subflow_recv_lookup(msk); 8956771bfd9SFlorian Westphal 8966771bfd9SFlorian Westphal if (!ssk) 8976771bfd9SFlorian Westphal break; 8986771bfd9SFlorian Westphal 8996771bfd9SFlorian Westphal lock_sock(ssk); 9006771bfd9SFlorian Westphal done = __mptcp_move_skbs_from_subflow(msk, ssk, &moved); 9016771bfd9SFlorian Westphal release_sock(ssk); 9026771bfd9SFlorian Westphal } while (!done); 9036771bfd9SFlorian Westphal 9046771bfd9SFlorian Westphal return moved > 0; 9056771bfd9SFlorian Westphal } 9066771bfd9SFlorian Westphal 907f870fa0bSMat Martineau static int mptcp_recvmsg(struct sock *sk, struct msghdr *msg, size_t len, 908f870fa0bSMat Martineau int nonblock, int flags, int *addr_len) 909f870fa0bSMat Martineau { 910f870fa0bSMat Martineau struct mptcp_sock *msk = mptcp_sk(sk); 911cec37a6eSPeter Krystad struct socket *ssock; 912cec37a6eSPeter Krystad int copied = 0; 9137a6a6cbcSPaolo Abeni int target; 9147a6a6cbcSPaolo Abeni long timeo; 915f870fa0bSMat Martineau 916f870fa0bSMat Martineau if (msg->msg_flags & ~(MSG_WAITALL | MSG_DONTWAIT)) 917f870fa0bSMat Martineau return -EOPNOTSUPP; 918f870fa0bSMat Martineau 919cec37a6eSPeter Krystad lock_sock(sk); 920cec37a6eSPeter Krystad ssock = __mptcp_tcp_fallback(msk); 9218ab183deSPaolo Abeni if (unlikely(ssock)) { 9228ab183deSPaolo Abeni fallback: 923e154659bSFlorian Westphal release_sock(sk); 924cec37a6eSPeter Krystad pr_debug("fallback-read subflow=%p", 925cec37a6eSPeter Krystad mptcp_subflow_ctx(ssock->sk)); 926cec37a6eSPeter Krystad copied = sock_recvmsg(ssock, msg, flags); 927cec37a6eSPeter Krystad return copied; 928cec37a6eSPeter Krystad } 929cec37a6eSPeter Krystad 9307a6a6cbcSPaolo Abeni timeo = sock_rcvtimeo(sk, nonblock); 9317a6a6cbcSPaolo Abeni 9327a6a6cbcSPaolo Abeni len = min_t(size_t, len, INT_MAX); 9337a6a6cbcSPaolo Abeni target = sock_rcvlowat(sk, flags & MSG_WAITALL, len); 934ec3edaa7SPeter Krystad __mptcp_flush_join_list(msk); 9357a6a6cbcSPaolo Abeni 9366771bfd9SFlorian Westphal while (len > (size_t)copied) { 9377a6a6cbcSPaolo Abeni int bytes_read; 9387a6a6cbcSPaolo Abeni 9396771bfd9SFlorian Westphal bytes_read = __mptcp_recvmsg_mskq(msk, msg, len - copied); 9406771bfd9SFlorian Westphal if (unlikely(bytes_read < 0)) { 9417a6a6cbcSPaolo Abeni if (!copied) 9427a6a6cbcSPaolo Abeni copied = bytes_read; 9436771bfd9SFlorian Westphal goto out_err; 944cec37a6eSPeter Krystad } 945cec37a6eSPeter Krystad 9467a6a6cbcSPaolo Abeni copied += bytes_read; 9477a6a6cbcSPaolo Abeni 9486771bfd9SFlorian Westphal if (skb_queue_empty(&sk->sk_receive_queue) && 9496771bfd9SFlorian Westphal __mptcp_move_skbs(msk)) 9506771bfd9SFlorian Westphal continue; 9517a6a6cbcSPaolo Abeni 9527a6a6cbcSPaolo Abeni /* only the master socket status is relevant here. The exit 9537a6a6cbcSPaolo Abeni * conditions mirror closely tcp_recvmsg() 9547a6a6cbcSPaolo Abeni */ 9557a6a6cbcSPaolo Abeni if (copied >= target) 9567a6a6cbcSPaolo Abeni break; 9577a6a6cbcSPaolo Abeni 9587a6a6cbcSPaolo Abeni if (copied) { 9597a6a6cbcSPaolo Abeni if (sk->sk_err || 9607a6a6cbcSPaolo Abeni sk->sk_state == TCP_CLOSE || 9617a6a6cbcSPaolo Abeni (sk->sk_shutdown & RCV_SHUTDOWN) || 9627a6a6cbcSPaolo Abeni !timeo || 9637a6a6cbcSPaolo Abeni signal_pending(current)) 9647a6a6cbcSPaolo Abeni break; 9657a6a6cbcSPaolo Abeni } else { 9667a6a6cbcSPaolo Abeni if (sk->sk_err) { 9677a6a6cbcSPaolo Abeni copied = sock_error(sk); 9687a6a6cbcSPaolo Abeni break; 9697a6a6cbcSPaolo Abeni } 9707a6a6cbcSPaolo Abeni 9717a6a6cbcSPaolo Abeni if (sk->sk_shutdown & RCV_SHUTDOWN) 9727a6a6cbcSPaolo Abeni break; 9737a6a6cbcSPaolo Abeni 9747a6a6cbcSPaolo Abeni if (sk->sk_state == TCP_CLOSE) { 9757a6a6cbcSPaolo Abeni copied = -ENOTCONN; 9767a6a6cbcSPaolo Abeni break; 9777a6a6cbcSPaolo Abeni } 9787a6a6cbcSPaolo Abeni 9797a6a6cbcSPaolo Abeni if (!timeo) { 9807a6a6cbcSPaolo Abeni copied = -EAGAIN; 9817a6a6cbcSPaolo Abeni break; 9827a6a6cbcSPaolo Abeni } 9837a6a6cbcSPaolo Abeni 9847a6a6cbcSPaolo Abeni if (signal_pending(current)) { 9857a6a6cbcSPaolo Abeni copied = sock_intr_errno(timeo); 9867a6a6cbcSPaolo Abeni break; 9877a6a6cbcSPaolo Abeni } 9887a6a6cbcSPaolo Abeni } 9897a6a6cbcSPaolo Abeni 9907a6a6cbcSPaolo Abeni pr_debug("block timeout %ld", timeo); 9917a6a6cbcSPaolo Abeni mptcp_wait_data(sk, &timeo); 9928ab183deSPaolo Abeni if (unlikely(__mptcp_tcp_fallback(msk))) 9938ab183deSPaolo Abeni goto fallback; 9947a6a6cbcSPaolo Abeni } 9957a6a6cbcSPaolo Abeni 9966771bfd9SFlorian Westphal if (skb_queue_empty(&sk->sk_receive_queue)) { 9976771bfd9SFlorian Westphal /* entire backlog drained, clear DATA_READY. */ 9987a6a6cbcSPaolo Abeni clear_bit(MPTCP_DATA_READY, &msk->flags); 9997a6a6cbcSPaolo Abeni 10006771bfd9SFlorian Westphal /* .. race-breaker: ssk might have gotten new data 10016771bfd9SFlorian Westphal * after last __mptcp_move_skbs() returned false. 10027a6a6cbcSPaolo Abeni */ 10036771bfd9SFlorian Westphal if (unlikely(__mptcp_move_skbs(msk))) 10046771bfd9SFlorian Westphal set_bit(MPTCP_DATA_READY, &msk->flags); 10056771bfd9SFlorian Westphal } else if (unlikely(!test_bit(MPTCP_DATA_READY, &msk->flags))) { 10066771bfd9SFlorian Westphal /* data to read but mptcp_wait_data() cleared DATA_READY */ 10077a6a6cbcSPaolo Abeni set_bit(MPTCP_DATA_READY, &msk->flags); 10087a6a6cbcSPaolo Abeni } 10096771bfd9SFlorian Westphal out_err: 1010cec37a6eSPeter Krystad release_sock(sk); 1011cec37a6eSPeter Krystad return copied; 1012cec37a6eSPeter Krystad } 1013cec37a6eSPeter Krystad 1014b51f9b80SPaolo Abeni static void mptcp_retransmit_handler(struct sock *sk) 1015b51f9b80SPaolo Abeni { 1016b51f9b80SPaolo Abeni struct mptcp_sock *msk = mptcp_sk(sk); 1017b51f9b80SPaolo Abeni 10183b1d6210SPaolo Abeni if (atomic64_read(&msk->snd_una) == msk->write_seq) { 1019b51f9b80SPaolo Abeni mptcp_stop_timer(sk); 10203b1d6210SPaolo Abeni } else { 10213b1d6210SPaolo Abeni set_bit(MPTCP_WORK_RTX, &msk->flags); 10223b1d6210SPaolo Abeni if (schedule_work(&msk->work)) 10233b1d6210SPaolo Abeni sock_hold(sk); 10243b1d6210SPaolo Abeni } 1025b51f9b80SPaolo Abeni } 1026b51f9b80SPaolo Abeni 1027b51f9b80SPaolo Abeni static void mptcp_retransmit_timer(struct timer_list *t) 1028b51f9b80SPaolo Abeni { 1029b51f9b80SPaolo Abeni struct inet_connection_sock *icsk = from_timer(icsk, t, 1030b51f9b80SPaolo Abeni icsk_retransmit_timer); 1031b51f9b80SPaolo Abeni struct sock *sk = &icsk->icsk_inet.sk; 1032b51f9b80SPaolo Abeni 1033b51f9b80SPaolo Abeni bh_lock_sock(sk); 1034b51f9b80SPaolo Abeni if (!sock_owned_by_user(sk)) { 1035b51f9b80SPaolo Abeni mptcp_retransmit_handler(sk); 1036b51f9b80SPaolo Abeni } else { 1037b51f9b80SPaolo Abeni /* delegate our work to tcp_release_cb() */ 1038b51f9b80SPaolo Abeni if (!test_and_set_bit(TCP_WRITE_TIMER_DEFERRED, 1039b51f9b80SPaolo Abeni &sk->sk_tsq_flags)) 1040b51f9b80SPaolo Abeni sock_hold(sk); 1041b51f9b80SPaolo Abeni } 1042b51f9b80SPaolo Abeni bh_unlock_sock(sk); 1043b51f9b80SPaolo Abeni sock_put(sk); 1044b51f9b80SPaolo Abeni } 1045b51f9b80SPaolo Abeni 10463b1d6210SPaolo Abeni /* Find an idle subflow. Return NULL if there is unacked data at tcp 10473b1d6210SPaolo Abeni * level. 10483b1d6210SPaolo Abeni * 10493b1d6210SPaolo Abeni * A backup subflow is returned only if that is the only kind available. 10503b1d6210SPaolo Abeni */ 10513b1d6210SPaolo Abeni static struct sock *mptcp_subflow_get_retrans(const struct mptcp_sock *msk) 10523b1d6210SPaolo Abeni { 10533b1d6210SPaolo Abeni struct mptcp_subflow_context *subflow; 10543b1d6210SPaolo Abeni struct sock *backup = NULL; 10553b1d6210SPaolo Abeni 10563b1d6210SPaolo Abeni sock_owned_by_me((const struct sock *)msk); 10573b1d6210SPaolo Abeni 10583b1d6210SPaolo Abeni mptcp_for_each_subflow(msk, subflow) { 10593b1d6210SPaolo Abeni struct sock *ssk = mptcp_subflow_tcp_sock(subflow); 10603b1d6210SPaolo Abeni 10613b1d6210SPaolo Abeni /* still data outstanding at TCP level? Don't retransmit. */ 10623b1d6210SPaolo Abeni if (!tcp_write_queue_empty(ssk)) 10633b1d6210SPaolo Abeni return NULL; 10643b1d6210SPaolo Abeni 10653b1d6210SPaolo Abeni if (subflow->backup) { 10663b1d6210SPaolo Abeni if (!backup) 10673b1d6210SPaolo Abeni backup = ssk; 10683b1d6210SPaolo Abeni continue; 10693b1d6210SPaolo Abeni } 10703b1d6210SPaolo Abeni 10713b1d6210SPaolo Abeni return ssk; 10723b1d6210SPaolo Abeni } 10733b1d6210SPaolo Abeni 10743b1d6210SPaolo Abeni return backup; 10753b1d6210SPaolo Abeni } 10763b1d6210SPaolo Abeni 1077cec37a6eSPeter Krystad /* subflow sockets can be either outgoing (connect) or incoming 1078cec37a6eSPeter Krystad * (accept). 1079cec37a6eSPeter Krystad * 1080cec37a6eSPeter Krystad * Outgoing subflows use in-kernel sockets. 1081cec37a6eSPeter Krystad * Incoming subflows do not have their own 'struct socket' allocated, 1082cec37a6eSPeter Krystad * so we need to use tcp_close() after detaching them from the mptcp 1083cec37a6eSPeter Krystad * parent socket. 1084cec37a6eSPeter Krystad */ 1085cec37a6eSPeter Krystad static void __mptcp_close_ssk(struct sock *sk, struct sock *ssk, 1086cec37a6eSPeter Krystad struct mptcp_subflow_context *subflow, 1087cec37a6eSPeter Krystad long timeout) 1088cec37a6eSPeter Krystad { 1089cec37a6eSPeter Krystad struct socket *sock = READ_ONCE(ssk->sk_socket); 1090cec37a6eSPeter Krystad 1091cec37a6eSPeter Krystad list_del(&subflow->node); 1092cec37a6eSPeter Krystad 1093cec37a6eSPeter Krystad if (sock && sock != sk->sk_socket) { 1094cec37a6eSPeter Krystad /* outgoing subflow */ 1095cec37a6eSPeter Krystad sock_release(sock); 1096cec37a6eSPeter Krystad } else { 1097cec37a6eSPeter Krystad /* incoming subflow */ 1098cec37a6eSPeter Krystad tcp_close(ssk, timeout); 1099cec37a6eSPeter Krystad } 1100f870fa0bSMat Martineau } 1101f870fa0bSMat Martineau 1102dc24f8b4SPaolo Abeni static unsigned int mptcp_sync_mss(struct sock *sk, u32 pmtu) 1103dc24f8b4SPaolo Abeni { 1104dc24f8b4SPaolo Abeni return 0; 1105dc24f8b4SPaolo Abeni } 1106dc24f8b4SPaolo Abeni 110759832e24SFlorian Westphal static void mptcp_check_for_eof(struct mptcp_sock *msk) 110859832e24SFlorian Westphal { 110959832e24SFlorian Westphal struct mptcp_subflow_context *subflow; 111059832e24SFlorian Westphal struct sock *sk = (struct sock *)msk; 111159832e24SFlorian Westphal int receivers = 0; 111259832e24SFlorian Westphal 111359832e24SFlorian Westphal mptcp_for_each_subflow(msk, subflow) 111459832e24SFlorian Westphal receivers += !subflow->rx_eof; 111559832e24SFlorian Westphal 111659832e24SFlorian Westphal if (!receivers && !(sk->sk_shutdown & RCV_SHUTDOWN)) { 111759832e24SFlorian Westphal /* hopefully temporary hack: propagate shutdown status 111859832e24SFlorian Westphal * to msk, when all subflows agree on it 111959832e24SFlorian Westphal */ 112059832e24SFlorian Westphal sk->sk_shutdown |= RCV_SHUTDOWN; 112159832e24SFlorian Westphal 112259832e24SFlorian Westphal smp_mb__before_atomic(); /* SHUTDOWN must be visible first */ 112359832e24SFlorian Westphal set_bit(MPTCP_DATA_READY, &msk->flags); 112459832e24SFlorian Westphal sk->sk_data_ready(sk); 112559832e24SFlorian Westphal } 112659832e24SFlorian Westphal } 112759832e24SFlorian Westphal 112880992017SPaolo Abeni static void mptcp_worker(struct work_struct *work) 112980992017SPaolo Abeni { 113080992017SPaolo Abeni struct mptcp_sock *msk = container_of(work, struct mptcp_sock, work); 11313b1d6210SPaolo Abeni struct sock *ssk, *sk = &msk->sk.icsk_inet.sk; 11323b1d6210SPaolo Abeni int orig_len, orig_offset, ret, mss_now = 0, size_goal = 0; 11333b1d6210SPaolo Abeni struct mptcp_data_frag *dfrag; 11343b1d6210SPaolo Abeni u64 orig_write_seq; 11353b1d6210SPaolo Abeni size_t copied = 0; 11363b1d6210SPaolo Abeni struct msghdr msg; 11373b1d6210SPaolo Abeni long timeo = 0; 113880992017SPaolo Abeni 113980992017SPaolo Abeni lock_sock(sk); 11403b1d6210SPaolo Abeni mptcp_clean_una(sk); 1141ec3edaa7SPeter Krystad __mptcp_flush_join_list(msk); 11426771bfd9SFlorian Westphal __mptcp_move_skbs(msk); 11433b1d6210SPaolo Abeni 114459832e24SFlorian Westphal if (test_and_clear_bit(MPTCP_WORK_EOF, &msk->flags)) 114559832e24SFlorian Westphal mptcp_check_for_eof(msk); 114659832e24SFlorian Westphal 11473b1d6210SPaolo Abeni if (!test_and_clear_bit(MPTCP_WORK_RTX, &msk->flags)) 11483b1d6210SPaolo Abeni goto unlock; 11493b1d6210SPaolo Abeni 11503b1d6210SPaolo Abeni dfrag = mptcp_rtx_head(sk); 11513b1d6210SPaolo Abeni if (!dfrag) 11523b1d6210SPaolo Abeni goto unlock; 11533b1d6210SPaolo Abeni 11543b1d6210SPaolo Abeni ssk = mptcp_subflow_get_retrans(msk); 11553b1d6210SPaolo Abeni if (!ssk) 11563b1d6210SPaolo Abeni goto reset_unlock; 11573b1d6210SPaolo Abeni 11583b1d6210SPaolo Abeni lock_sock(ssk); 11593b1d6210SPaolo Abeni 11603b1d6210SPaolo Abeni msg.msg_flags = MSG_DONTWAIT; 11613b1d6210SPaolo Abeni orig_len = dfrag->data_len; 11623b1d6210SPaolo Abeni orig_offset = dfrag->offset; 11633b1d6210SPaolo Abeni orig_write_seq = dfrag->data_seq; 11643b1d6210SPaolo Abeni while (dfrag->data_len > 0) { 11653b1d6210SPaolo Abeni ret = mptcp_sendmsg_frag(sk, ssk, &msg, dfrag, &timeo, &mss_now, 11663b1d6210SPaolo Abeni &size_goal); 11673b1d6210SPaolo Abeni if (ret < 0) 11683b1d6210SPaolo Abeni break; 11693b1d6210SPaolo Abeni 1170fc518953SFlorian Westphal MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_RETRANSSEGS); 11713b1d6210SPaolo Abeni copied += ret; 11723b1d6210SPaolo Abeni dfrag->data_len -= ret; 11733b1d6210SPaolo Abeni dfrag->offset += ret; 11743b1d6210SPaolo Abeni } 11753b1d6210SPaolo Abeni if (copied) 11763b1d6210SPaolo Abeni tcp_push(ssk, msg.msg_flags, mss_now, tcp_sk(ssk)->nonagle, 11773b1d6210SPaolo Abeni size_goal); 11783b1d6210SPaolo Abeni 11793b1d6210SPaolo Abeni dfrag->data_seq = orig_write_seq; 11803b1d6210SPaolo Abeni dfrag->offset = orig_offset; 11813b1d6210SPaolo Abeni dfrag->data_len = orig_len; 11823b1d6210SPaolo Abeni 11833b1d6210SPaolo Abeni mptcp_set_timeout(sk, ssk); 11843b1d6210SPaolo Abeni release_sock(ssk); 11853b1d6210SPaolo Abeni 11863b1d6210SPaolo Abeni reset_unlock: 11873b1d6210SPaolo Abeni if (!mptcp_timer_pending(sk)) 11883b1d6210SPaolo Abeni mptcp_reset_timer(sk); 11893b1d6210SPaolo Abeni 11903b1d6210SPaolo Abeni unlock: 119180992017SPaolo Abeni release_sock(sk); 119280992017SPaolo Abeni sock_put(sk); 119380992017SPaolo Abeni } 119480992017SPaolo Abeni 1195784325e9SMatthieu Baerts static int __mptcp_init_sock(struct sock *sk) 1196f870fa0bSMat Martineau { 1197cec37a6eSPeter Krystad struct mptcp_sock *msk = mptcp_sk(sk); 1198cec37a6eSPeter Krystad 1199ec3edaa7SPeter Krystad spin_lock_init(&msk->join_list_lock); 1200ec3edaa7SPeter Krystad 1201cec37a6eSPeter Krystad INIT_LIST_HEAD(&msk->conn_list); 1202ec3edaa7SPeter Krystad INIT_LIST_HEAD(&msk->join_list); 120318b683bfSPaolo Abeni INIT_LIST_HEAD(&msk->rtx_queue); 12041891c4a0SFlorian Westphal __set_bit(MPTCP_SEND_SPACE, &msk->flags); 120580992017SPaolo Abeni INIT_WORK(&msk->work, mptcp_worker); 1206cec37a6eSPeter Krystad 12078ab183deSPaolo Abeni msk->first = NULL; 1208dc24f8b4SPaolo Abeni inet_csk(sk)->icsk_sync_mss = mptcp_sync_mss; 12098ab183deSPaolo Abeni 12101b1c7a0eSPeter Krystad mptcp_pm_data_init(msk); 12111b1c7a0eSPeter Krystad 1212b51f9b80SPaolo Abeni /* re-use the csk retrans timer for MPTCP-level retrans */ 1213b51f9b80SPaolo Abeni timer_setup(&msk->sk.icsk_retransmit_timer, mptcp_retransmit_timer, 0); 1214b51f9b80SPaolo Abeni 1215f870fa0bSMat Martineau return 0; 1216f870fa0bSMat Martineau } 1217f870fa0bSMat Martineau 1218784325e9SMatthieu Baerts static int mptcp_init_sock(struct sock *sk) 1219784325e9SMatthieu Baerts { 1220fc518953SFlorian Westphal struct net *net = sock_net(sk); 1221fc518953SFlorian Westphal int ret; 122218b683bfSPaolo Abeni 1223fc518953SFlorian Westphal if (!mptcp_is_enabled(net)) 1224fc518953SFlorian Westphal return -ENOPROTOOPT; 1225fc518953SFlorian Westphal 1226fc518953SFlorian Westphal if (unlikely(!net->mib.mptcp_statistics) && !mptcp_mib_alloc(net)) 1227fc518953SFlorian Westphal return -ENOMEM; 1228fc518953SFlorian Westphal 1229fc518953SFlorian Westphal ret = __mptcp_init_sock(sk); 123018b683bfSPaolo Abeni if (ret) 123118b683bfSPaolo Abeni return ret; 123218b683bfSPaolo Abeni 1233d027236cSPaolo Abeni sk_sockets_allocated_inc(sk); 12343f8e0aaeSPaolo Abeni sk->sk_sndbuf = sock_net(sk)->ipv4.sysctl_tcp_wmem[2]; 1235d027236cSPaolo Abeni 123618b683bfSPaolo Abeni return 0; 123718b683bfSPaolo Abeni } 123818b683bfSPaolo Abeni 123918b683bfSPaolo Abeni static void __mptcp_clear_xmit(struct sock *sk) 124018b683bfSPaolo Abeni { 124118b683bfSPaolo Abeni struct mptcp_sock *msk = mptcp_sk(sk); 124218b683bfSPaolo Abeni struct mptcp_data_frag *dtmp, *dfrag; 124318b683bfSPaolo Abeni 1244b51f9b80SPaolo Abeni sk_stop_timer(sk, &msk->sk.icsk_retransmit_timer); 1245b51f9b80SPaolo Abeni 124618b683bfSPaolo Abeni list_for_each_entry_safe(dfrag, dtmp, &msk->rtx_queue, list) 1247d027236cSPaolo Abeni dfrag_clear(sk, dfrag); 1248784325e9SMatthieu Baerts } 1249784325e9SMatthieu Baerts 125080992017SPaolo Abeni static void mptcp_cancel_work(struct sock *sk) 125180992017SPaolo Abeni { 125280992017SPaolo Abeni struct mptcp_sock *msk = mptcp_sk(sk); 125380992017SPaolo Abeni 125480992017SPaolo Abeni if (cancel_work_sync(&msk->work)) 125580992017SPaolo Abeni sock_put(sk); 125680992017SPaolo Abeni } 125780992017SPaolo Abeni 125876c42a29SMat Martineau static void mptcp_subflow_shutdown(struct sock *ssk, int how, 125976c42a29SMat Martineau bool data_fin_tx_enable, u64 data_fin_tx_seq) 126021498490SPeter Krystad { 126121498490SPeter Krystad lock_sock(ssk); 126221498490SPeter Krystad 126321498490SPeter Krystad switch (ssk->sk_state) { 126421498490SPeter Krystad case TCP_LISTEN: 126521498490SPeter Krystad if (!(how & RCV_SHUTDOWN)) 126621498490SPeter Krystad break; 126721498490SPeter Krystad /* fall through */ 126821498490SPeter Krystad case TCP_SYN_SENT: 126921498490SPeter Krystad tcp_disconnect(ssk, O_NONBLOCK); 127021498490SPeter Krystad break; 127121498490SPeter Krystad default: 127276c42a29SMat Martineau if (data_fin_tx_enable) { 127376c42a29SMat Martineau struct mptcp_subflow_context *subflow; 127476c42a29SMat Martineau 127576c42a29SMat Martineau subflow = mptcp_subflow_ctx(ssk); 127676c42a29SMat Martineau subflow->data_fin_tx_seq = data_fin_tx_seq; 127776c42a29SMat Martineau subflow->data_fin_tx_enable = 1; 127876c42a29SMat Martineau } 127976c42a29SMat Martineau 128021498490SPeter Krystad ssk->sk_shutdown |= how; 128121498490SPeter Krystad tcp_shutdown(ssk, how); 128221498490SPeter Krystad break; 128321498490SPeter Krystad } 128421498490SPeter Krystad 128521498490SPeter Krystad /* Wake up anyone sleeping in poll. */ 128621498490SPeter Krystad ssk->sk_state_change(ssk); 128721498490SPeter Krystad release_sock(ssk); 128821498490SPeter Krystad } 128921498490SPeter Krystad 12908ab183deSPaolo Abeni /* Called with msk lock held, releases such lock before returning */ 12912c22c06cSFlorian Westphal static void mptcp_close(struct sock *sk, long timeout) 1292f870fa0bSMat Martineau { 1293cec37a6eSPeter Krystad struct mptcp_subflow_context *subflow, *tmp; 1294f870fa0bSMat Martineau struct mptcp_sock *msk = mptcp_sk(sk); 1295b2c5b614SFlorian Westphal LIST_HEAD(conn_list); 129676c42a29SMat Martineau u64 data_fin_tx_seq; 1297f870fa0bSMat Martineau 12982c22c06cSFlorian Westphal lock_sock(sk); 12992c22c06cSFlorian Westphal 130079c0949eSPeter Krystad mptcp_token_destroy(msk->token); 1301f870fa0bSMat Martineau inet_sk_state_store(sk, TCP_CLOSE); 1302f870fa0bSMat Martineau 1303ec3edaa7SPeter Krystad __mptcp_flush_join_list(msk); 1304ec3edaa7SPeter Krystad 1305b2c5b614SFlorian Westphal list_splice_init(&msk->conn_list, &conn_list); 1306b2c5b614SFlorian Westphal 130776c42a29SMat Martineau data_fin_tx_seq = msk->write_seq; 130876c42a29SMat Martineau 130918b683bfSPaolo Abeni __mptcp_clear_xmit(sk); 131018b683bfSPaolo Abeni 1311b2c5b614SFlorian Westphal release_sock(sk); 1312b2c5b614SFlorian Westphal 1313b2c5b614SFlorian Westphal list_for_each_entry_safe(subflow, tmp, &conn_list, node) { 1314cec37a6eSPeter Krystad struct sock *ssk = mptcp_subflow_tcp_sock(subflow); 1315cec37a6eSPeter Krystad 131676c42a29SMat Martineau subflow->data_fin_tx_seq = data_fin_tx_seq; 131776c42a29SMat Martineau subflow->data_fin_tx_enable = 1; 1318cec37a6eSPeter Krystad __mptcp_close_ssk(sk, ssk, subflow, timeout); 1319f870fa0bSMat Martineau } 1320f870fa0bSMat Martineau 132180992017SPaolo Abeni mptcp_cancel_work(sk); 1322926bdeabSPeter Krystad mptcp_pm_close(msk); 132380992017SPaolo Abeni 13246771bfd9SFlorian Westphal __skb_queue_purge(&sk->sk_receive_queue); 13256771bfd9SFlorian Westphal 1326cec37a6eSPeter Krystad sk_common_release(sk); 1327f870fa0bSMat Martineau } 1328f870fa0bSMat Martineau 1329cf7da0d6SPeter Krystad static void mptcp_copy_inaddrs(struct sock *msk, const struct sock *ssk) 1330cf7da0d6SPeter Krystad { 1331cf7da0d6SPeter Krystad #if IS_ENABLED(CONFIG_MPTCP_IPV6) 1332cf7da0d6SPeter Krystad const struct ipv6_pinfo *ssk6 = inet6_sk(ssk); 1333cf7da0d6SPeter Krystad struct ipv6_pinfo *msk6 = inet6_sk(msk); 1334cf7da0d6SPeter Krystad 1335cf7da0d6SPeter Krystad msk->sk_v6_daddr = ssk->sk_v6_daddr; 1336cf7da0d6SPeter Krystad msk->sk_v6_rcv_saddr = ssk->sk_v6_rcv_saddr; 1337cf7da0d6SPeter Krystad 1338cf7da0d6SPeter Krystad if (msk6 && ssk6) { 1339cf7da0d6SPeter Krystad msk6->saddr = ssk6->saddr; 1340cf7da0d6SPeter Krystad msk6->flow_label = ssk6->flow_label; 1341cf7da0d6SPeter Krystad } 1342cf7da0d6SPeter Krystad #endif 1343cf7da0d6SPeter Krystad 1344cf7da0d6SPeter Krystad inet_sk(msk)->inet_num = inet_sk(ssk)->inet_num; 1345cf7da0d6SPeter Krystad inet_sk(msk)->inet_dport = inet_sk(ssk)->inet_dport; 1346cf7da0d6SPeter Krystad inet_sk(msk)->inet_sport = inet_sk(ssk)->inet_sport; 1347cf7da0d6SPeter Krystad inet_sk(msk)->inet_daddr = inet_sk(ssk)->inet_daddr; 1348cf7da0d6SPeter Krystad inet_sk(msk)->inet_saddr = inet_sk(ssk)->inet_saddr; 1349cf7da0d6SPeter Krystad inet_sk(msk)->inet_rcv_saddr = inet_sk(ssk)->inet_rcv_saddr; 1350cf7da0d6SPeter Krystad } 1351cf7da0d6SPeter Krystad 135218b683bfSPaolo Abeni static int mptcp_disconnect(struct sock *sk, int flags) 135318b683bfSPaolo Abeni { 135442c556feSFlorian Westphal /* Should never be called. 135542c556feSFlorian Westphal * inet_stream_connect() calls ->disconnect, but that 135642c556feSFlorian Westphal * refers to the subflow socket, not the mptcp one. 135742c556feSFlorian Westphal */ 135842c556feSFlorian Westphal WARN_ON_ONCE(1); 135942c556feSFlorian Westphal return 0; 136018b683bfSPaolo Abeni } 136118b683bfSPaolo Abeni 1362b0519de8SFlorian Westphal #if IS_ENABLED(CONFIG_MPTCP_IPV6) 1363b0519de8SFlorian Westphal static struct ipv6_pinfo *mptcp_inet6_sk(const struct sock *sk) 1364b0519de8SFlorian Westphal { 1365b0519de8SFlorian Westphal unsigned int offset = sizeof(struct mptcp6_sock) - sizeof(struct ipv6_pinfo); 1366b0519de8SFlorian Westphal 1367b0519de8SFlorian Westphal return (struct ipv6_pinfo *)(((u8 *)sk) + offset); 1368b0519de8SFlorian Westphal } 1369b0519de8SFlorian Westphal #endif 1370b0519de8SFlorian Westphal 1371fca5c82cSPaolo Abeni struct sock *mptcp_sk_clone(const struct sock *sk, 1372cfde141eSPaolo Abeni const struct mptcp_options_received *mp_opt, 1373fca5c82cSPaolo Abeni struct request_sock *req) 1374b0519de8SFlorian Westphal { 137558b09919SPaolo Abeni struct mptcp_subflow_request_sock *subflow_req = mptcp_subflow_rsk(req); 1376b0519de8SFlorian Westphal struct sock *nsk = sk_clone_lock(sk, GFP_ATOMIC); 137758b09919SPaolo Abeni struct mptcp_sock *msk; 137858b09919SPaolo Abeni u64 ack_seq; 1379b0519de8SFlorian Westphal 1380b0519de8SFlorian Westphal if (!nsk) 1381b0519de8SFlorian Westphal return NULL; 1382b0519de8SFlorian Westphal 1383b0519de8SFlorian Westphal #if IS_ENABLED(CONFIG_MPTCP_IPV6) 1384b0519de8SFlorian Westphal if (nsk->sk_family == AF_INET6) 1385b0519de8SFlorian Westphal inet_sk(nsk)->pinet6 = mptcp_inet6_sk(nsk); 1386b0519de8SFlorian Westphal #endif 1387b0519de8SFlorian Westphal 138858b09919SPaolo Abeni __mptcp_init_sock(nsk); 138958b09919SPaolo Abeni 139058b09919SPaolo Abeni msk = mptcp_sk(nsk); 139158b09919SPaolo Abeni msk->local_key = subflow_req->local_key; 139258b09919SPaolo Abeni msk->token = subflow_req->token; 139358b09919SPaolo Abeni msk->subflow = NULL; 139458b09919SPaolo Abeni 139558b09919SPaolo Abeni if (unlikely(mptcp_token_new_accept(subflow_req->token, nsk))) { 13969f5ca6a5SFlorian Westphal nsk->sk_state = TCP_CLOSE; 139758b09919SPaolo Abeni bh_unlock_sock(nsk); 139858b09919SPaolo Abeni 139958b09919SPaolo Abeni /* we can't call into mptcp_close() here - possible BH context 14009f5ca6a5SFlorian Westphal * free the sock directly. 14019f5ca6a5SFlorian Westphal * sk_clone_lock() sets nsk refcnt to two, hence call sk_free() 14029f5ca6a5SFlorian Westphal * too. 140358b09919SPaolo Abeni */ 14049f5ca6a5SFlorian Westphal sk_common_release(nsk); 140558b09919SPaolo Abeni sk_free(nsk); 140658b09919SPaolo Abeni return NULL; 140758b09919SPaolo Abeni } 140858b09919SPaolo Abeni 140958b09919SPaolo Abeni msk->write_seq = subflow_req->idsn + 1; 1410cc9d2566SPaolo Abeni atomic64_set(&msk->snd_una, msk->write_seq); 1411cfde141eSPaolo Abeni if (mp_opt->mp_capable) { 141258b09919SPaolo Abeni msk->can_ack = true; 1413cfde141eSPaolo Abeni msk->remote_key = mp_opt->sndr_key; 141458b09919SPaolo Abeni mptcp_crypto_key_sha(msk->remote_key, NULL, &ack_seq); 141558b09919SPaolo Abeni ack_seq++; 141658b09919SPaolo Abeni msk->ack_seq = ack_seq; 141758b09919SPaolo Abeni } 14187f20d5fcSPaolo Abeni 14195e20087dSFlorian Westphal sock_reset_flag(nsk, SOCK_RCU_FREE); 14207f20d5fcSPaolo Abeni /* will be fully established after successful MPC subflow creation */ 14217f20d5fcSPaolo Abeni inet_sk_state_store(nsk, TCP_SYN_RECV); 142258b09919SPaolo Abeni bh_unlock_sock(nsk); 142358b09919SPaolo Abeni 142458b09919SPaolo Abeni /* keep a single reference */ 142558b09919SPaolo Abeni __sock_put(nsk); 1426b0519de8SFlorian Westphal return nsk; 1427b0519de8SFlorian Westphal } 1428b0519de8SFlorian Westphal 1429cf7da0d6SPeter Krystad static struct sock *mptcp_accept(struct sock *sk, int flags, int *err, 1430cf7da0d6SPeter Krystad bool kern) 1431cf7da0d6SPeter Krystad { 1432cf7da0d6SPeter Krystad struct mptcp_sock *msk = mptcp_sk(sk); 1433cf7da0d6SPeter Krystad struct socket *listener; 1434cf7da0d6SPeter Krystad struct sock *newsk; 1435cf7da0d6SPeter Krystad 1436cf7da0d6SPeter Krystad listener = __mptcp_nmpc_socket(msk); 1437cf7da0d6SPeter Krystad if (WARN_ON_ONCE(!listener)) { 1438cf7da0d6SPeter Krystad *err = -EINVAL; 1439cf7da0d6SPeter Krystad return NULL; 1440cf7da0d6SPeter Krystad } 1441cf7da0d6SPeter Krystad 1442cf7da0d6SPeter Krystad pr_debug("msk=%p, listener=%p", msk, mptcp_subflow_ctx(listener->sk)); 1443cf7da0d6SPeter Krystad newsk = inet_csk_accept(listener->sk, flags, err, kern); 1444cf7da0d6SPeter Krystad if (!newsk) 1445cf7da0d6SPeter Krystad return NULL; 1446cf7da0d6SPeter Krystad 1447cf7da0d6SPeter Krystad pr_debug("msk=%p, subflow is mptcp=%d", msk, sk_is_mptcp(newsk)); 1448cf7da0d6SPeter Krystad 1449cf7da0d6SPeter Krystad if (sk_is_mptcp(newsk)) { 1450cf7da0d6SPeter Krystad struct mptcp_subflow_context *subflow; 1451cf7da0d6SPeter Krystad struct sock *new_mptcp_sock; 1452cf7da0d6SPeter Krystad struct sock *ssk = newsk; 1453cf7da0d6SPeter Krystad 1454cf7da0d6SPeter Krystad subflow = mptcp_subflow_ctx(newsk); 145558b09919SPaolo Abeni new_mptcp_sock = subflow->conn; 145658b09919SPaolo Abeni 145758b09919SPaolo Abeni /* is_mptcp should be false if subflow->conn is missing, see 145858b09919SPaolo Abeni * subflow_syn_recv_sock() 145958b09919SPaolo Abeni */ 146058b09919SPaolo Abeni if (WARN_ON_ONCE(!new_mptcp_sock)) { 146158b09919SPaolo Abeni tcp_sk(newsk)->is_mptcp = 0; 146258b09919SPaolo Abeni return newsk; 146358b09919SPaolo Abeni } 146458b09919SPaolo Abeni 146558b09919SPaolo Abeni /* acquire the 2nd reference for the owning socket */ 146658b09919SPaolo Abeni sock_hold(new_mptcp_sock); 1467cf7da0d6SPeter Krystad 1468cf7da0d6SPeter Krystad local_bh_disable(); 146958b09919SPaolo Abeni bh_lock_sock(new_mptcp_sock); 1470cf7da0d6SPeter Krystad msk = mptcp_sk(new_mptcp_sock); 14718ab183deSPaolo Abeni msk->first = newsk; 1472cf7da0d6SPeter Krystad 1473cf7da0d6SPeter Krystad newsk = new_mptcp_sock; 1474cf7da0d6SPeter Krystad mptcp_copy_inaddrs(newsk, ssk); 1475cf7da0d6SPeter Krystad list_add(&subflow->node, &msk->conn_list); 1476df1036daSFlorian Westphal inet_sk_state_store(newsk, TCP_ESTABLISHED); 1477cf7da0d6SPeter Krystad 1478cf7da0d6SPeter Krystad bh_unlock_sock(new_mptcp_sock); 1479fc518953SFlorian Westphal 1480fc518953SFlorian Westphal __MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_MPCAPABLEPASSIVEACK); 1481cf7da0d6SPeter Krystad local_bh_enable(); 1482fc518953SFlorian Westphal } else { 1483fc518953SFlorian Westphal MPTCP_INC_STATS(sock_net(sk), 1484fc518953SFlorian Westphal MPTCP_MIB_MPCAPABLEPASSIVEFALLBACK); 1485cf7da0d6SPeter Krystad } 1486cf7da0d6SPeter Krystad 1487cf7da0d6SPeter Krystad return newsk; 1488cf7da0d6SPeter Krystad } 1489cf7da0d6SPeter Krystad 149079c0949eSPeter Krystad static void mptcp_destroy(struct sock *sk) 149179c0949eSPeter Krystad { 1492c9fd9c5fSFlorian Westphal struct mptcp_sock *msk = mptcp_sk(sk); 1493c9fd9c5fSFlorian Westphal 1494c9fd9c5fSFlorian Westphal if (msk->cached_ext) 1495c9fd9c5fSFlorian Westphal __skb_ext_put(msk->cached_ext); 1496d027236cSPaolo Abeni 1497d027236cSPaolo Abeni sk_sockets_allocated_dec(sk); 149879c0949eSPeter Krystad } 149979c0949eSPeter Krystad 1500717e79c8SPeter Krystad static int mptcp_setsockopt(struct sock *sk, int level, int optname, 150150e741bbSFlorian Westphal char __user *optval, unsigned int optlen) 1502717e79c8SPeter Krystad { 1503717e79c8SPeter Krystad struct mptcp_sock *msk = mptcp_sk(sk); 1504717e79c8SPeter Krystad struct socket *ssock; 1505717e79c8SPeter Krystad 1506717e79c8SPeter Krystad pr_debug("msk=%p", msk); 1507717e79c8SPeter Krystad 1508717e79c8SPeter Krystad /* @@ the meaning of setsockopt() when the socket is connected and 1509b6e4a1aeSMat Martineau * there are multiple subflows is not yet defined. It is up to the 1510b6e4a1aeSMat Martineau * MPTCP-level socket to configure the subflows until the subflow 1511b6e4a1aeSMat Martineau * is in TCP fallback, when TCP socket options are passed through 1512b6e4a1aeSMat Martineau * to the one remaining subflow. 1513717e79c8SPeter Krystad */ 1514717e79c8SPeter Krystad lock_sock(sk); 1515b6e4a1aeSMat Martineau ssock = __mptcp_tcp_fallback(msk); 1516e154659bSFlorian Westphal release_sock(sk); 1517b6e4a1aeSMat Martineau if (ssock) 1518b6e4a1aeSMat Martineau return tcp_setsockopt(ssock->sk, level, optname, optval, 1519b6e4a1aeSMat Martineau optlen); 152050e741bbSFlorian Westphal 1521b6e4a1aeSMat Martineau return -EOPNOTSUPP; 1522717e79c8SPeter Krystad } 1523717e79c8SPeter Krystad 1524717e79c8SPeter Krystad static int mptcp_getsockopt(struct sock *sk, int level, int optname, 152550e741bbSFlorian Westphal char __user *optval, int __user *option) 1526717e79c8SPeter Krystad { 1527717e79c8SPeter Krystad struct mptcp_sock *msk = mptcp_sk(sk); 1528717e79c8SPeter Krystad struct socket *ssock; 1529717e79c8SPeter Krystad 1530717e79c8SPeter Krystad pr_debug("msk=%p", msk); 1531717e79c8SPeter Krystad 1532b6e4a1aeSMat Martineau /* @@ the meaning of setsockopt() when the socket is connected and 1533b6e4a1aeSMat Martineau * there are multiple subflows is not yet defined. It is up to the 1534b6e4a1aeSMat Martineau * MPTCP-level socket to configure the subflows until the subflow 1535b6e4a1aeSMat Martineau * is in TCP fallback, when socket options are passed through 1536b6e4a1aeSMat Martineau * to the one remaining subflow. 1537717e79c8SPeter Krystad */ 1538717e79c8SPeter Krystad lock_sock(sk); 1539b6e4a1aeSMat Martineau ssock = __mptcp_tcp_fallback(msk); 1540e154659bSFlorian Westphal release_sock(sk); 1541b6e4a1aeSMat Martineau if (ssock) 1542b6e4a1aeSMat Martineau return tcp_getsockopt(ssock->sk, level, optname, optval, 1543b6e4a1aeSMat Martineau option); 154450e741bbSFlorian Westphal 1545b6e4a1aeSMat Martineau return -EOPNOTSUPP; 1546717e79c8SPeter Krystad } 1547717e79c8SPeter Krystad 1548b51f9b80SPaolo Abeni #define MPTCP_DEFERRED_ALL (TCPF_DELACK_TIMER_DEFERRED | \ 1549b51f9b80SPaolo Abeni TCPF_WRITE_TIMER_DEFERRED) 155014c441b5SPaolo Abeni 155114c441b5SPaolo Abeni /* this is very alike tcp_release_cb() but we must handle differently a 155214c441b5SPaolo Abeni * different set of events 155314c441b5SPaolo Abeni */ 155414c441b5SPaolo Abeni static void mptcp_release_cb(struct sock *sk) 155514c441b5SPaolo Abeni { 155614c441b5SPaolo Abeni unsigned long flags, nflags; 155714c441b5SPaolo Abeni 155814c441b5SPaolo Abeni do { 155914c441b5SPaolo Abeni flags = sk->sk_tsq_flags; 156014c441b5SPaolo Abeni if (!(flags & MPTCP_DEFERRED_ALL)) 156114c441b5SPaolo Abeni return; 156214c441b5SPaolo Abeni nflags = flags & ~MPTCP_DEFERRED_ALL; 156314c441b5SPaolo Abeni } while (cmpxchg(&sk->sk_tsq_flags, flags, nflags) != flags); 156414c441b5SPaolo Abeni 1565b51f9b80SPaolo Abeni sock_release_ownership(sk); 1566b51f9b80SPaolo Abeni 156714c441b5SPaolo Abeni if (flags & TCPF_DELACK_TIMER_DEFERRED) { 156814c441b5SPaolo Abeni struct mptcp_sock *msk = mptcp_sk(sk); 156914c441b5SPaolo Abeni struct sock *ssk; 157014c441b5SPaolo Abeni 157114c441b5SPaolo Abeni ssk = mptcp_subflow_recv_lookup(msk); 157214c441b5SPaolo Abeni if (!ssk || !schedule_work(&msk->work)) 157314c441b5SPaolo Abeni __sock_put(sk); 157414c441b5SPaolo Abeni } 1575b51f9b80SPaolo Abeni 1576b51f9b80SPaolo Abeni if (flags & TCPF_WRITE_TIMER_DEFERRED) { 1577b51f9b80SPaolo Abeni mptcp_retransmit_handler(sk); 1578b51f9b80SPaolo Abeni __sock_put(sk); 1579b51f9b80SPaolo Abeni } 158014c441b5SPaolo Abeni } 158114c441b5SPaolo Abeni 1582cec37a6eSPeter Krystad static int mptcp_get_port(struct sock *sk, unsigned short snum) 1583f870fa0bSMat Martineau { 1584f870fa0bSMat Martineau struct mptcp_sock *msk = mptcp_sk(sk); 1585cec37a6eSPeter Krystad struct socket *ssock; 1586f870fa0bSMat Martineau 1587cec37a6eSPeter Krystad ssock = __mptcp_nmpc_socket(msk); 1588cec37a6eSPeter Krystad pr_debug("msk=%p, subflow=%p", msk, ssock); 1589cec37a6eSPeter Krystad if (WARN_ON_ONCE(!ssock)) 1590cec37a6eSPeter Krystad return -EINVAL; 1591f870fa0bSMat Martineau 1592cec37a6eSPeter Krystad return inet_csk_get_port(ssock->sk, snum); 1593cec37a6eSPeter Krystad } 1594f870fa0bSMat Martineau 1595cec37a6eSPeter Krystad void mptcp_finish_connect(struct sock *ssk) 1596cec37a6eSPeter Krystad { 1597cec37a6eSPeter Krystad struct mptcp_subflow_context *subflow; 1598cec37a6eSPeter Krystad struct mptcp_sock *msk; 1599cec37a6eSPeter Krystad struct sock *sk; 16006d0060f6SMat Martineau u64 ack_seq; 1601f870fa0bSMat Martineau 1602cec37a6eSPeter Krystad subflow = mptcp_subflow_ctx(ssk); 1603cec37a6eSPeter Krystad sk = subflow->conn; 1604cec37a6eSPeter Krystad msk = mptcp_sk(sk); 1605cec37a6eSPeter Krystad 1606fc518953SFlorian Westphal if (!subflow->mp_capable) { 1607fc518953SFlorian Westphal MPTCP_INC_STATS(sock_net(sk), 1608fc518953SFlorian Westphal MPTCP_MIB_MPCAPABLEACTIVEFALLBACK); 1609fc518953SFlorian Westphal return; 1610fc518953SFlorian Westphal } 1611fc518953SFlorian Westphal 1612648ef4b8SMat Martineau pr_debug("msk=%p, token=%u", sk, subflow->token); 1613648ef4b8SMat Martineau 16146d0060f6SMat Martineau mptcp_crypto_key_sha(subflow->remote_key, NULL, &ack_seq); 16156d0060f6SMat Martineau ack_seq++; 1616648ef4b8SMat Martineau subflow->map_seq = ack_seq; 1617648ef4b8SMat Martineau subflow->map_subflow_seq = 1; 16186d0060f6SMat Martineau subflow->rel_write_seq = 1; 16196d0060f6SMat Martineau 1620cec37a6eSPeter Krystad /* the socket is not connected yet, no msk/subflow ops can access/race 1621cec37a6eSPeter Krystad * accessing the field below 1622cec37a6eSPeter Krystad */ 1623cec37a6eSPeter Krystad WRITE_ONCE(msk->remote_key, subflow->remote_key); 1624cec37a6eSPeter Krystad WRITE_ONCE(msk->local_key, subflow->local_key); 162579c0949eSPeter Krystad WRITE_ONCE(msk->token, subflow->token); 16266d0060f6SMat Martineau WRITE_ONCE(msk->write_seq, subflow->idsn + 1); 16276d0060f6SMat Martineau WRITE_ONCE(msk->ack_seq, ack_seq); 1628d22f4988SChristoph Paasch WRITE_ONCE(msk->can_ack, 1); 1629cc9d2566SPaolo Abeni atomic64_set(&msk->snd_una, msk->write_seq); 16301b1c7a0eSPeter Krystad 16311b1c7a0eSPeter Krystad mptcp_pm_new_connection(msk, 0); 1632f870fa0bSMat Martineau } 1633f870fa0bSMat Martineau 1634cf7da0d6SPeter Krystad static void mptcp_sock_graft(struct sock *sk, struct socket *parent) 1635cf7da0d6SPeter Krystad { 1636cf7da0d6SPeter Krystad write_lock_bh(&sk->sk_callback_lock); 1637cf7da0d6SPeter Krystad rcu_assign_pointer(sk->sk_wq, &parent->wq); 1638cf7da0d6SPeter Krystad sk_set_socket(sk, parent); 1639cf7da0d6SPeter Krystad sk->sk_uid = SOCK_INODE(parent)->i_uid; 1640cf7da0d6SPeter Krystad write_unlock_bh(&sk->sk_callback_lock); 1641cf7da0d6SPeter Krystad } 1642cf7da0d6SPeter Krystad 1643f296234cSPeter Krystad bool mptcp_finish_join(struct sock *sk) 1644f296234cSPeter Krystad { 1645f296234cSPeter Krystad struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(sk); 1646f296234cSPeter Krystad struct mptcp_sock *msk = mptcp_sk(subflow->conn); 1647f296234cSPeter Krystad struct sock *parent = (void *)msk; 1648f296234cSPeter Krystad struct socket *parent_sock; 1649ec3edaa7SPeter Krystad bool ret; 1650f296234cSPeter Krystad 1651f296234cSPeter Krystad pr_debug("msk=%p, subflow=%p", msk, subflow); 1652f296234cSPeter Krystad 1653f296234cSPeter Krystad /* mptcp socket already closing? */ 1654f296234cSPeter Krystad if (inet_sk_state_load(parent) != TCP_ESTABLISHED) 1655f296234cSPeter Krystad return false; 1656f296234cSPeter Krystad 1657f296234cSPeter Krystad if (!msk->pm.server_side) 1658f296234cSPeter Krystad return true; 1659f296234cSPeter Krystad 1660f296234cSPeter Krystad /* passive connection, attach to msk socket */ 1661f296234cSPeter Krystad parent_sock = READ_ONCE(parent->sk_socket); 1662f296234cSPeter Krystad if (parent_sock && !sk->sk_socket) 1663f296234cSPeter Krystad mptcp_sock_graft(sk, parent_sock); 1664f296234cSPeter Krystad 1665ec3edaa7SPeter Krystad ret = mptcp_pm_allow_new_subflow(msk); 1666ec3edaa7SPeter Krystad if (ret) { 166764d950aeSChristoph Paasch subflow->map_seq = msk->ack_seq; 166864d950aeSChristoph Paasch 1669ec3edaa7SPeter Krystad /* active connections are already on conn_list */ 1670ec3edaa7SPeter Krystad spin_lock_bh(&msk->join_list_lock); 1671ec3edaa7SPeter Krystad if (!WARN_ON_ONCE(!list_empty(&subflow->node))) 1672ec3edaa7SPeter Krystad list_add_tail(&subflow->node, &msk->join_list); 1673ec3edaa7SPeter Krystad spin_unlock_bh(&msk->join_list_lock); 1674ec3edaa7SPeter Krystad } 1675ec3edaa7SPeter Krystad return ret; 1676f296234cSPeter Krystad } 1677f296234cSPeter Krystad 16781891c4a0SFlorian Westphal static bool mptcp_memory_free(const struct sock *sk, int wake) 16791891c4a0SFlorian Westphal { 16801891c4a0SFlorian Westphal struct mptcp_sock *msk = mptcp_sk(sk); 16811891c4a0SFlorian Westphal 16821891c4a0SFlorian Westphal return wake ? test_bit(MPTCP_SEND_SPACE, &msk->flags) : true; 16831891c4a0SFlorian Westphal } 16841891c4a0SFlorian Westphal 1685f870fa0bSMat Martineau static struct proto mptcp_prot = { 1686f870fa0bSMat Martineau .name = "MPTCP", 1687f870fa0bSMat Martineau .owner = THIS_MODULE, 1688f870fa0bSMat Martineau .init = mptcp_init_sock, 168918b683bfSPaolo Abeni .disconnect = mptcp_disconnect, 1690f870fa0bSMat Martineau .close = mptcp_close, 1691cf7da0d6SPeter Krystad .accept = mptcp_accept, 1692717e79c8SPeter Krystad .setsockopt = mptcp_setsockopt, 1693717e79c8SPeter Krystad .getsockopt = mptcp_getsockopt, 1694f870fa0bSMat Martineau .shutdown = tcp_shutdown, 169579c0949eSPeter Krystad .destroy = mptcp_destroy, 1696f870fa0bSMat Martineau .sendmsg = mptcp_sendmsg, 1697f870fa0bSMat Martineau .recvmsg = mptcp_recvmsg, 169814c441b5SPaolo Abeni .release_cb = mptcp_release_cb, 1699f870fa0bSMat Martineau .hash = inet_hash, 1700f870fa0bSMat Martineau .unhash = inet_unhash, 1701cec37a6eSPeter Krystad .get_port = mptcp_get_port, 1702d027236cSPaolo Abeni .sockets_allocated = &mptcp_sockets_allocated, 1703d027236cSPaolo Abeni .memory_allocated = &tcp_memory_allocated, 1704d027236cSPaolo Abeni .memory_pressure = &tcp_memory_pressure, 17051891c4a0SFlorian Westphal .stream_memory_free = mptcp_memory_free, 1706d027236cSPaolo Abeni .sysctl_wmem_offset = offsetof(struct net, ipv4.sysctl_tcp_wmem), 1707d027236cSPaolo Abeni .sysctl_mem = sysctl_tcp_mem, 1708f870fa0bSMat Martineau .obj_size = sizeof(struct mptcp_sock), 1709f870fa0bSMat Martineau .no_autobind = true, 1710f870fa0bSMat Martineau }; 1711f870fa0bSMat Martineau 17122303f994SPeter Krystad static int mptcp_bind(struct socket *sock, struct sockaddr *uaddr, int addr_len) 17132303f994SPeter Krystad { 17142303f994SPeter Krystad struct mptcp_sock *msk = mptcp_sk(sock->sk); 17152303f994SPeter Krystad struct socket *ssock; 1716cf7da0d6SPeter Krystad int err; 17172303f994SPeter Krystad 17182303f994SPeter Krystad lock_sock(sock->sk); 17192303f994SPeter Krystad ssock = __mptcp_socket_create(msk, MPTCP_SAME_STATE); 17202303f994SPeter Krystad if (IS_ERR(ssock)) { 17212303f994SPeter Krystad err = PTR_ERR(ssock); 17222303f994SPeter Krystad goto unlock; 17232303f994SPeter Krystad } 17242303f994SPeter Krystad 17252303f994SPeter Krystad err = ssock->ops->bind(ssock, uaddr, addr_len); 1726cf7da0d6SPeter Krystad if (!err) 1727cf7da0d6SPeter Krystad mptcp_copy_inaddrs(sock->sk, ssock->sk); 17282303f994SPeter Krystad 17292303f994SPeter Krystad unlock: 17302303f994SPeter Krystad release_sock(sock->sk); 17312303f994SPeter Krystad return err; 17322303f994SPeter Krystad } 17332303f994SPeter Krystad 17342303f994SPeter Krystad static int mptcp_stream_connect(struct socket *sock, struct sockaddr *uaddr, 17352303f994SPeter Krystad int addr_len, int flags) 17362303f994SPeter Krystad { 17372303f994SPeter Krystad struct mptcp_sock *msk = mptcp_sk(sock->sk); 17382303f994SPeter Krystad struct socket *ssock; 17392303f994SPeter Krystad int err; 17402303f994SPeter Krystad 17412303f994SPeter Krystad lock_sock(sock->sk); 17422303f994SPeter Krystad ssock = __mptcp_socket_create(msk, TCP_SYN_SENT); 17432303f994SPeter Krystad if (IS_ERR(ssock)) { 17442303f994SPeter Krystad err = PTR_ERR(ssock); 17452303f994SPeter Krystad goto unlock; 17462303f994SPeter Krystad } 17472303f994SPeter Krystad 1748cf7da0d6SPeter Krystad #ifdef CONFIG_TCP_MD5SIG 1749cf7da0d6SPeter Krystad /* no MPTCP if MD5SIG is enabled on this socket or we may run out of 1750cf7da0d6SPeter Krystad * TCP option space. 1751cf7da0d6SPeter Krystad */ 1752cf7da0d6SPeter Krystad if (rcu_access_pointer(tcp_sk(ssock->sk)->md5sig_info)) 1753cf7da0d6SPeter Krystad mptcp_subflow_ctx(ssock->sk)->request_mptcp = 0; 1754cf7da0d6SPeter Krystad #endif 1755cf7da0d6SPeter Krystad 17562303f994SPeter Krystad err = ssock->ops->connect(ssock, uaddr, addr_len, flags); 17572303f994SPeter Krystad inet_sk_state_store(sock->sk, inet_sk_state_load(ssock->sk)); 1758cf7da0d6SPeter Krystad mptcp_copy_inaddrs(sock->sk, ssock->sk); 17592303f994SPeter Krystad 17602303f994SPeter Krystad unlock: 17612303f994SPeter Krystad release_sock(sock->sk); 17622303f994SPeter Krystad return err; 17632303f994SPeter Krystad } 17642303f994SPeter Krystad 1765cf7da0d6SPeter Krystad static int mptcp_v4_getname(struct socket *sock, struct sockaddr *uaddr, 1766cf7da0d6SPeter Krystad int peer) 1767cf7da0d6SPeter Krystad { 1768cf7da0d6SPeter Krystad if (sock->sk->sk_prot == &tcp_prot) { 1769cf7da0d6SPeter Krystad /* we are being invoked from __sys_accept4, after 1770cf7da0d6SPeter Krystad * mptcp_accept() has just accepted a non-mp-capable 1771cf7da0d6SPeter Krystad * flow: sk is a tcp_sk, not an mptcp one. 1772cf7da0d6SPeter Krystad * 1773cf7da0d6SPeter Krystad * Hand the socket over to tcp so all further socket ops 1774cf7da0d6SPeter Krystad * bypass mptcp. 1775cf7da0d6SPeter Krystad */ 1776cf7da0d6SPeter Krystad sock->ops = &inet_stream_ops; 1777cf7da0d6SPeter Krystad } 1778cf7da0d6SPeter Krystad 1779cf7da0d6SPeter Krystad return inet_getname(sock, uaddr, peer); 1780cf7da0d6SPeter Krystad } 1781cf7da0d6SPeter Krystad 1782cf7da0d6SPeter Krystad #if IS_ENABLED(CONFIG_MPTCP_IPV6) 1783cf7da0d6SPeter Krystad static int mptcp_v6_getname(struct socket *sock, struct sockaddr *uaddr, 1784cf7da0d6SPeter Krystad int peer) 1785cf7da0d6SPeter Krystad { 1786cf7da0d6SPeter Krystad if (sock->sk->sk_prot == &tcpv6_prot) { 1787cf7da0d6SPeter Krystad /* we are being invoked from __sys_accept4 after 1788cf7da0d6SPeter Krystad * mptcp_accept() has accepted a non-mp-capable 1789cf7da0d6SPeter Krystad * subflow: sk is a tcp_sk, not mptcp. 1790cf7da0d6SPeter Krystad * 1791cf7da0d6SPeter Krystad * Hand the socket over to tcp so all further 1792cf7da0d6SPeter Krystad * socket ops bypass mptcp. 1793cf7da0d6SPeter Krystad */ 1794cf7da0d6SPeter Krystad sock->ops = &inet6_stream_ops; 1795cf7da0d6SPeter Krystad } 1796cf7da0d6SPeter Krystad 1797cf7da0d6SPeter Krystad return inet6_getname(sock, uaddr, peer); 1798cf7da0d6SPeter Krystad } 1799cf7da0d6SPeter Krystad #endif 1800cf7da0d6SPeter Krystad 1801cf7da0d6SPeter Krystad static int mptcp_listen(struct socket *sock, int backlog) 1802cf7da0d6SPeter Krystad { 1803cf7da0d6SPeter Krystad struct mptcp_sock *msk = mptcp_sk(sock->sk); 1804cf7da0d6SPeter Krystad struct socket *ssock; 1805cf7da0d6SPeter Krystad int err; 1806cf7da0d6SPeter Krystad 1807cf7da0d6SPeter Krystad pr_debug("msk=%p", msk); 1808cf7da0d6SPeter Krystad 1809cf7da0d6SPeter Krystad lock_sock(sock->sk); 1810cf7da0d6SPeter Krystad ssock = __mptcp_socket_create(msk, TCP_LISTEN); 1811cf7da0d6SPeter Krystad if (IS_ERR(ssock)) { 1812cf7da0d6SPeter Krystad err = PTR_ERR(ssock); 1813cf7da0d6SPeter Krystad goto unlock; 1814cf7da0d6SPeter Krystad } 1815cf7da0d6SPeter Krystad 18165e20087dSFlorian Westphal sock_set_flag(sock->sk, SOCK_RCU_FREE); 18175e20087dSFlorian Westphal 1818cf7da0d6SPeter Krystad err = ssock->ops->listen(ssock, backlog); 1819cf7da0d6SPeter Krystad inet_sk_state_store(sock->sk, inet_sk_state_load(ssock->sk)); 1820cf7da0d6SPeter Krystad if (!err) 1821cf7da0d6SPeter Krystad mptcp_copy_inaddrs(sock->sk, ssock->sk); 1822cf7da0d6SPeter Krystad 1823cf7da0d6SPeter Krystad unlock: 1824cf7da0d6SPeter Krystad release_sock(sock->sk); 1825cf7da0d6SPeter Krystad return err; 1826cf7da0d6SPeter Krystad } 1827cf7da0d6SPeter Krystad 1828cf7da0d6SPeter Krystad static bool is_tcp_proto(const struct proto *p) 1829cf7da0d6SPeter Krystad { 1830cf7da0d6SPeter Krystad #if IS_ENABLED(CONFIG_MPTCP_IPV6) 1831cf7da0d6SPeter Krystad return p == &tcp_prot || p == &tcpv6_prot; 1832cf7da0d6SPeter Krystad #else 1833cf7da0d6SPeter Krystad return p == &tcp_prot; 1834cf7da0d6SPeter Krystad #endif 1835cf7da0d6SPeter Krystad } 1836cf7da0d6SPeter Krystad 1837cf7da0d6SPeter Krystad static int mptcp_stream_accept(struct socket *sock, struct socket *newsock, 1838cf7da0d6SPeter Krystad int flags, bool kern) 1839cf7da0d6SPeter Krystad { 1840cf7da0d6SPeter Krystad struct mptcp_sock *msk = mptcp_sk(sock->sk); 1841cf7da0d6SPeter Krystad struct socket *ssock; 1842cf7da0d6SPeter Krystad int err; 1843cf7da0d6SPeter Krystad 1844cf7da0d6SPeter Krystad pr_debug("msk=%p", msk); 1845cf7da0d6SPeter Krystad 1846cf7da0d6SPeter Krystad lock_sock(sock->sk); 1847cf7da0d6SPeter Krystad if (sock->sk->sk_state != TCP_LISTEN) 1848cf7da0d6SPeter Krystad goto unlock_fail; 1849cf7da0d6SPeter Krystad 1850cf7da0d6SPeter Krystad ssock = __mptcp_nmpc_socket(msk); 1851cf7da0d6SPeter Krystad if (!ssock) 1852cf7da0d6SPeter Krystad goto unlock_fail; 1853cf7da0d6SPeter Krystad 1854cf7da0d6SPeter Krystad sock_hold(ssock->sk); 1855cf7da0d6SPeter Krystad release_sock(sock->sk); 1856cf7da0d6SPeter Krystad 1857cf7da0d6SPeter Krystad err = ssock->ops->accept(sock, newsock, flags, kern); 1858cf7da0d6SPeter Krystad if (err == 0 && !is_tcp_proto(newsock->sk->sk_prot)) { 1859cf7da0d6SPeter Krystad struct mptcp_sock *msk = mptcp_sk(newsock->sk); 1860cf7da0d6SPeter Krystad struct mptcp_subflow_context *subflow; 1861cf7da0d6SPeter Krystad 1862cf7da0d6SPeter Krystad /* set ssk->sk_socket of accept()ed flows to mptcp socket. 1863cf7da0d6SPeter Krystad * This is needed so NOSPACE flag can be set from tcp stack. 1864cf7da0d6SPeter Krystad */ 1865ec3edaa7SPeter Krystad __mptcp_flush_join_list(msk); 1866cf7da0d6SPeter Krystad list_for_each_entry(subflow, &msk->conn_list, node) { 1867cf7da0d6SPeter Krystad struct sock *ssk = mptcp_subflow_tcp_sock(subflow); 1868cf7da0d6SPeter Krystad 1869cf7da0d6SPeter Krystad if (!ssk->sk_socket) 1870cf7da0d6SPeter Krystad mptcp_sock_graft(ssk, newsock); 1871cf7da0d6SPeter Krystad } 1872cf7da0d6SPeter Krystad } 1873cf7da0d6SPeter Krystad 1874cf7da0d6SPeter Krystad sock_put(ssock->sk); 1875cf7da0d6SPeter Krystad return err; 1876cf7da0d6SPeter Krystad 1877cf7da0d6SPeter Krystad unlock_fail: 1878cf7da0d6SPeter Krystad release_sock(sock->sk); 1879cf7da0d6SPeter Krystad return -EINVAL; 1880cf7da0d6SPeter Krystad } 1881cf7da0d6SPeter Krystad 18822303f994SPeter Krystad static __poll_t mptcp_poll(struct file *file, struct socket *sock, 18832303f994SPeter Krystad struct poll_table_struct *wait) 18842303f994SPeter Krystad { 18851891c4a0SFlorian Westphal struct sock *sk = sock->sk; 18868ab183deSPaolo Abeni struct mptcp_sock *msk; 18871891c4a0SFlorian Westphal struct socket *ssock; 18882303f994SPeter Krystad __poll_t mask = 0; 18892303f994SPeter Krystad 18901891c4a0SFlorian Westphal msk = mptcp_sk(sk); 18911891c4a0SFlorian Westphal lock_sock(sk); 18920b4f33deSFlorian Westphal ssock = __mptcp_tcp_fallback(msk); 18930b4f33deSFlorian Westphal if (!ssock) 18941891c4a0SFlorian Westphal ssock = __mptcp_nmpc_socket(msk); 18951891c4a0SFlorian Westphal if (ssock) { 18961891c4a0SFlorian Westphal mask = ssock->ops->poll(file, ssock, wait); 18971891c4a0SFlorian Westphal release_sock(sk); 18981891c4a0SFlorian Westphal return mask; 18991891c4a0SFlorian Westphal } 19001891c4a0SFlorian Westphal 19011891c4a0SFlorian Westphal release_sock(sk); 19021891c4a0SFlorian Westphal sock_poll_wait(file, sock, wait); 19031891c4a0SFlorian Westphal lock_sock(sk); 19041891c4a0SFlorian Westphal 19051891c4a0SFlorian Westphal if (test_bit(MPTCP_DATA_READY, &msk->flags)) 19061891c4a0SFlorian Westphal mask = EPOLLIN | EPOLLRDNORM; 19071891c4a0SFlorian Westphal if (sk_stream_is_writeable(sk) && 19081891c4a0SFlorian Westphal test_bit(MPTCP_SEND_SPACE, &msk->flags)) 19091891c4a0SFlorian Westphal mask |= EPOLLOUT | EPOLLWRNORM; 19101891c4a0SFlorian Westphal if (sk->sk_shutdown & RCV_SHUTDOWN) 19111891c4a0SFlorian Westphal mask |= EPOLLIN | EPOLLRDNORM | EPOLLRDHUP; 19121891c4a0SFlorian Westphal 19131891c4a0SFlorian Westphal release_sock(sk); 19141891c4a0SFlorian Westphal 19152303f994SPeter Krystad return mask; 19162303f994SPeter Krystad } 19172303f994SPeter Krystad 191821498490SPeter Krystad static int mptcp_shutdown(struct socket *sock, int how) 191921498490SPeter Krystad { 192021498490SPeter Krystad struct mptcp_sock *msk = mptcp_sk(sock->sk); 192121498490SPeter Krystad struct mptcp_subflow_context *subflow; 19220b4f33deSFlorian Westphal struct socket *ssock; 192321498490SPeter Krystad int ret = 0; 192421498490SPeter Krystad 192521498490SPeter Krystad pr_debug("sk=%p, how=%d", msk, how); 192621498490SPeter Krystad 192721498490SPeter Krystad lock_sock(sock->sk); 19280b4f33deSFlorian Westphal ssock = __mptcp_tcp_fallback(msk); 19290b4f33deSFlorian Westphal if (ssock) { 19300b4f33deSFlorian Westphal release_sock(sock->sk); 19310b4f33deSFlorian Westphal return inet_shutdown(ssock, how); 19320b4f33deSFlorian Westphal } 193321498490SPeter Krystad 193421498490SPeter Krystad if (how == SHUT_WR || how == SHUT_RDWR) 193521498490SPeter Krystad inet_sk_state_store(sock->sk, TCP_FIN_WAIT1); 193621498490SPeter Krystad 193721498490SPeter Krystad how++; 193821498490SPeter Krystad 193921498490SPeter Krystad if ((how & ~SHUTDOWN_MASK) || !how) { 194021498490SPeter Krystad ret = -EINVAL; 194121498490SPeter Krystad goto out_unlock; 194221498490SPeter Krystad } 194321498490SPeter Krystad 194421498490SPeter Krystad if (sock->state == SS_CONNECTING) { 194521498490SPeter Krystad if ((1 << sock->sk->sk_state) & 194621498490SPeter Krystad (TCPF_SYN_SENT | TCPF_SYN_RECV | TCPF_CLOSE)) 194721498490SPeter Krystad sock->state = SS_DISCONNECTING; 194821498490SPeter Krystad else 194921498490SPeter Krystad sock->state = SS_CONNECTED; 195021498490SPeter Krystad } 195121498490SPeter Krystad 1952ec3edaa7SPeter Krystad __mptcp_flush_join_list(msk); 195321498490SPeter Krystad mptcp_for_each_subflow(msk, subflow) { 195421498490SPeter Krystad struct sock *tcp_sk = mptcp_subflow_tcp_sock(subflow); 195521498490SPeter Krystad 195676c42a29SMat Martineau mptcp_subflow_shutdown(tcp_sk, how, 1, msk->write_seq); 195721498490SPeter Krystad } 195821498490SPeter Krystad 195921498490SPeter Krystad out_unlock: 196021498490SPeter Krystad release_sock(sock->sk); 196121498490SPeter Krystad 196221498490SPeter Krystad return ret; 196321498490SPeter Krystad } 196421498490SPeter Krystad 1965e42f1ac6SFlorian Westphal static const struct proto_ops mptcp_stream_ops = { 1966e42f1ac6SFlorian Westphal .family = PF_INET, 1967e42f1ac6SFlorian Westphal .owner = THIS_MODULE, 1968e42f1ac6SFlorian Westphal .release = inet_release, 1969e42f1ac6SFlorian Westphal .bind = mptcp_bind, 1970e42f1ac6SFlorian Westphal .connect = mptcp_stream_connect, 1971e42f1ac6SFlorian Westphal .socketpair = sock_no_socketpair, 1972e42f1ac6SFlorian Westphal .accept = mptcp_stream_accept, 1973e42f1ac6SFlorian Westphal .getname = mptcp_v4_getname, 1974e42f1ac6SFlorian Westphal .poll = mptcp_poll, 1975e42f1ac6SFlorian Westphal .ioctl = inet_ioctl, 1976e42f1ac6SFlorian Westphal .gettstamp = sock_gettstamp, 1977e42f1ac6SFlorian Westphal .listen = mptcp_listen, 1978e42f1ac6SFlorian Westphal .shutdown = mptcp_shutdown, 1979e42f1ac6SFlorian Westphal .setsockopt = sock_common_setsockopt, 1980e42f1ac6SFlorian Westphal .getsockopt = sock_common_getsockopt, 1981e42f1ac6SFlorian Westphal .sendmsg = inet_sendmsg, 1982e42f1ac6SFlorian Westphal .recvmsg = inet_recvmsg, 1983e42f1ac6SFlorian Westphal .mmap = sock_no_mmap, 1984e42f1ac6SFlorian Westphal .sendpage = inet_sendpage, 1985e42f1ac6SFlorian Westphal #ifdef CONFIG_COMPAT 1986e42f1ac6SFlorian Westphal .compat_setsockopt = compat_sock_common_setsockopt, 1987e42f1ac6SFlorian Westphal .compat_getsockopt = compat_sock_common_getsockopt, 1988e42f1ac6SFlorian Westphal #endif 1989e42f1ac6SFlorian Westphal }; 19902303f994SPeter Krystad 1991f870fa0bSMat Martineau static struct inet_protosw mptcp_protosw = { 1992f870fa0bSMat Martineau .type = SOCK_STREAM, 1993f870fa0bSMat Martineau .protocol = IPPROTO_MPTCP, 1994f870fa0bSMat Martineau .prot = &mptcp_prot, 19952303f994SPeter Krystad .ops = &mptcp_stream_ops, 19962303f994SPeter Krystad .flags = INET_PROTOSW_ICSK, 1997f870fa0bSMat Martineau }; 1998f870fa0bSMat Martineau 1999784325e9SMatthieu Baerts void mptcp_proto_init(void) 2000f870fa0bSMat Martineau { 20012303f994SPeter Krystad mptcp_prot.h.hashinfo = tcp_prot.h.hashinfo; 20022303f994SPeter Krystad 2003d027236cSPaolo Abeni if (percpu_counter_init(&mptcp_sockets_allocated, 0, GFP_KERNEL)) 2004d027236cSPaolo Abeni panic("Failed to allocate MPTCP pcpu counter\n"); 2005d027236cSPaolo Abeni 20062303f994SPeter Krystad mptcp_subflow_init(); 20071b1c7a0eSPeter Krystad mptcp_pm_init(); 20082303f994SPeter Krystad 2009f870fa0bSMat Martineau if (proto_register(&mptcp_prot, 1) != 0) 2010f870fa0bSMat Martineau panic("Failed to register MPTCP proto.\n"); 2011f870fa0bSMat Martineau 2012f870fa0bSMat Martineau inet_register_protosw(&mptcp_protosw); 20136771bfd9SFlorian Westphal 20146771bfd9SFlorian Westphal BUILD_BUG_ON(sizeof(struct mptcp_skb_cb) > sizeof_field(struct sk_buff, cb)); 2015f870fa0bSMat Martineau } 2016f870fa0bSMat Martineau 2017f870fa0bSMat Martineau #if IS_ENABLED(CONFIG_MPTCP_IPV6) 2018e42f1ac6SFlorian Westphal static const struct proto_ops mptcp_v6_stream_ops = { 2019e42f1ac6SFlorian Westphal .family = PF_INET6, 2020e42f1ac6SFlorian Westphal .owner = THIS_MODULE, 2021e42f1ac6SFlorian Westphal .release = inet6_release, 2022e42f1ac6SFlorian Westphal .bind = mptcp_bind, 2023e42f1ac6SFlorian Westphal .connect = mptcp_stream_connect, 2024e42f1ac6SFlorian Westphal .socketpair = sock_no_socketpair, 2025e42f1ac6SFlorian Westphal .accept = mptcp_stream_accept, 2026e42f1ac6SFlorian Westphal .getname = mptcp_v6_getname, 2027e42f1ac6SFlorian Westphal .poll = mptcp_poll, 2028e42f1ac6SFlorian Westphal .ioctl = inet6_ioctl, 2029e42f1ac6SFlorian Westphal .gettstamp = sock_gettstamp, 2030e42f1ac6SFlorian Westphal .listen = mptcp_listen, 2031e42f1ac6SFlorian Westphal .shutdown = mptcp_shutdown, 2032e42f1ac6SFlorian Westphal .setsockopt = sock_common_setsockopt, 2033e42f1ac6SFlorian Westphal .getsockopt = sock_common_getsockopt, 2034e42f1ac6SFlorian Westphal .sendmsg = inet6_sendmsg, 2035e42f1ac6SFlorian Westphal .recvmsg = inet6_recvmsg, 2036e42f1ac6SFlorian Westphal .mmap = sock_no_mmap, 2037e42f1ac6SFlorian Westphal .sendpage = inet_sendpage, 2038e42f1ac6SFlorian Westphal #ifdef CONFIG_COMPAT 2039e42f1ac6SFlorian Westphal .compat_setsockopt = compat_sock_common_setsockopt, 2040e42f1ac6SFlorian Westphal .compat_getsockopt = compat_sock_common_getsockopt, 2041e42f1ac6SFlorian Westphal #endif 2042e42f1ac6SFlorian Westphal }; 2043e42f1ac6SFlorian Westphal 2044f870fa0bSMat Martineau static struct proto mptcp_v6_prot; 2045f870fa0bSMat Martineau 204679c0949eSPeter Krystad static void mptcp_v6_destroy(struct sock *sk) 204779c0949eSPeter Krystad { 204879c0949eSPeter Krystad mptcp_destroy(sk); 204979c0949eSPeter Krystad inet6_destroy_sock(sk); 205079c0949eSPeter Krystad } 205179c0949eSPeter Krystad 2052f870fa0bSMat Martineau static struct inet_protosw mptcp_v6_protosw = { 2053f870fa0bSMat Martineau .type = SOCK_STREAM, 2054f870fa0bSMat Martineau .protocol = IPPROTO_MPTCP, 2055f870fa0bSMat Martineau .prot = &mptcp_v6_prot, 20562303f994SPeter Krystad .ops = &mptcp_v6_stream_ops, 2057f870fa0bSMat Martineau .flags = INET_PROTOSW_ICSK, 2058f870fa0bSMat Martineau }; 2059f870fa0bSMat Martineau 2060784325e9SMatthieu Baerts int mptcp_proto_v6_init(void) 2061f870fa0bSMat Martineau { 2062f870fa0bSMat Martineau int err; 2063f870fa0bSMat Martineau 2064f870fa0bSMat Martineau mptcp_v6_prot = mptcp_prot; 2065f870fa0bSMat Martineau strcpy(mptcp_v6_prot.name, "MPTCPv6"); 2066f870fa0bSMat Martineau mptcp_v6_prot.slab = NULL; 206779c0949eSPeter Krystad mptcp_v6_prot.destroy = mptcp_v6_destroy; 2068b0519de8SFlorian Westphal mptcp_v6_prot.obj_size = sizeof(struct mptcp6_sock); 2069f870fa0bSMat Martineau 2070f870fa0bSMat Martineau err = proto_register(&mptcp_v6_prot, 1); 2071f870fa0bSMat Martineau if (err) 2072f870fa0bSMat Martineau return err; 2073f870fa0bSMat Martineau 2074f870fa0bSMat Martineau err = inet6_register_protosw(&mptcp_v6_protosw); 2075f870fa0bSMat Martineau if (err) 2076f870fa0bSMat Martineau proto_unregister(&mptcp_v6_prot); 2077f870fa0bSMat Martineau 2078f870fa0bSMat Martineau return err; 2079f870fa0bSMat Martineau } 2080f870fa0bSMat Martineau #endif 2081