1f870fa0bSMat Martineau // SPDX-License-Identifier: GPL-2.0 2f870fa0bSMat Martineau /* Multipath TCP 3f870fa0bSMat Martineau * 4f870fa0bSMat Martineau * Copyright (c) 2017 - 2019, Intel Corporation. 5f870fa0bSMat Martineau */ 6f870fa0bSMat Martineau 7f870fa0bSMat Martineau #define pr_fmt(fmt) "MPTCP: " fmt 8f870fa0bSMat Martineau 9f870fa0bSMat Martineau #include <linux/kernel.h> 10f870fa0bSMat Martineau #include <linux/module.h> 11f870fa0bSMat Martineau #include <linux/netdevice.h> 127a6a6cbcSPaolo Abeni #include <linux/sched/signal.h> 137a6a6cbcSPaolo Abeni #include <linux/atomic.h> 14f870fa0bSMat Martineau #include <net/sock.h> 15f870fa0bSMat Martineau #include <net/inet_common.h> 16f870fa0bSMat Martineau #include <net/inet_hashtables.h> 17f870fa0bSMat Martineau #include <net/protocol.h> 18f870fa0bSMat Martineau #include <net/tcp.h> 193721b9b6SMat Martineau #include <net/tcp_states.h> 20cf7da0d6SPeter Krystad #if IS_ENABLED(CONFIG_MPTCP_IPV6) 21cf7da0d6SPeter Krystad #include <net/transp_v6.h> 22cf7da0d6SPeter Krystad #endif 23f870fa0bSMat Martineau #include <net/mptcp.h> 24f870fa0bSMat Martineau #include "protocol.h" 25fc518953SFlorian Westphal #include "mib.h" 26f870fa0bSMat Martineau 27b0519de8SFlorian Westphal #if IS_ENABLED(CONFIG_MPTCP_IPV6) 28b0519de8SFlorian Westphal struct mptcp6_sock { 29b0519de8SFlorian Westphal struct mptcp_sock msk; 30b0519de8SFlorian Westphal struct ipv6_pinfo np; 31b0519de8SFlorian Westphal }; 32b0519de8SFlorian Westphal #endif 33b0519de8SFlorian Westphal 346771bfd9SFlorian Westphal struct mptcp_skb_cb { 356771bfd9SFlorian Westphal u32 offset; 366771bfd9SFlorian Westphal }; 376771bfd9SFlorian Westphal 386771bfd9SFlorian Westphal #define MPTCP_SKB_CB(__skb) ((struct mptcp_skb_cb *)&((__skb)->cb[0])) 396771bfd9SFlorian Westphal 40d027236cSPaolo Abeni static struct percpu_counter mptcp_sockets_allocated; 41d027236cSPaolo Abeni 422303f994SPeter Krystad /* If msk has an initial subflow socket, and the MP_CAPABLE handshake has not 432303f994SPeter Krystad * completed yet or has failed, return the subflow socket. 442303f994SPeter Krystad * Otherwise return NULL. 452303f994SPeter Krystad */ 462303f994SPeter Krystad static struct socket *__mptcp_nmpc_socket(const struct mptcp_sock *msk) 472303f994SPeter Krystad { 48d22f4988SChristoph Paasch if (!msk->subflow || READ_ONCE(msk->can_ack)) 492303f994SPeter Krystad return NULL; 502303f994SPeter Krystad 512303f994SPeter Krystad return msk->subflow; 522303f994SPeter Krystad } 532303f994SPeter Krystad 54d2f77c53SPaolo Abeni static bool mptcp_is_tcpsk(struct sock *sk) 550b4f33deSFlorian Westphal { 560b4f33deSFlorian Westphal struct socket *sock = sk->sk_socket; 570b4f33deSFlorian Westphal 580b4f33deSFlorian Westphal if (unlikely(sk->sk_prot == &tcp_prot)) { 590b4f33deSFlorian Westphal /* we are being invoked after mptcp_accept() has 600b4f33deSFlorian Westphal * accepted a non-mp-capable flow: sk is a tcp_sk, 610b4f33deSFlorian Westphal * not an mptcp one. 620b4f33deSFlorian Westphal * 630b4f33deSFlorian Westphal * Hand the socket over to tcp so all further socket ops 640b4f33deSFlorian Westphal * bypass mptcp. 650b4f33deSFlorian Westphal */ 660b4f33deSFlorian Westphal sock->ops = &inet_stream_ops; 67d2f77c53SPaolo Abeni return true; 680b4f33deSFlorian Westphal #if IS_ENABLED(CONFIG_MPTCP_IPV6) 690b4f33deSFlorian Westphal } else if (unlikely(sk->sk_prot == &tcpv6_prot)) { 700b4f33deSFlorian Westphal sock->ops = &inet6_stream_ops; 71d2f77c53SPaolo Abeni return true; 720b4f33deSFlorian Westphal #endif 730b4f33deSFlorian Westphal } 740b4f33deSFlorian Westphal 75d2f77c53SPaolo Abeni return false; 760b4f33deSFlorian Westphal } 770b4f33deSFlorian Westphal 7876660afbSPaolo Abeni static struct sock *__mptcp_tcp_fallback(struct mptcp_sock *msk) 79cec37a6eSPeter Krystad { 80cec37a6eSPeter Krystad sock_owned_by_me((const struct sock *)msk); 81cec37a6eSPeter Krystad 82e1ff9e82SDavide Caratti if (likely(!__mptcp_check_fallback(msk))) 83cec37a6eSPeter Krystad return NULL; 84cec37a6eSPeter Krystad 8576660afbSPaolo Abeni return msk->first; 868ab183deSPaolo Abeni } 878ab183deSPaolo Abeni 88fa68018dSPaolo Abeni static int __mptcp_socket_create(struct mptcp_sock *msk) 892303f994SPeter Krystad { 902303f994SPeter Krystad struct mptcp_subflow_context *subflow; 912303f994SPeter Krystad struct sock *sk = (struct sock *)msk; 922303f994SPeter Krystad struct socket *ssock; 932303f994SPeter Krystad int err; 942303f994SPeter Krystad 952303f994SPeter Krystad err = mptcp_subflow_create_socket(sk, &ssock); 962303f994SPeter Krystad if (err) 97fa68018dSPaolo Abeni return err; 982303f994SPeter Krystad 998ab183deSPaolo Abeni msk->first = ssock->sk; 1002303f994SPeter Krystad msk->subflow = ssock; 1012303f994SPeter Krystad subflow = mptcp_subflow_ctx(ssock->sk); 102cec37a6eSPeter Krystad list_add(&subflow->node, &msk->conn_list); 1032303f994SPeter Krystad subflow->request_mptcp = 1; 1042303f994SPeter Krystad 105e1ff9e82SDavide Caratti /* accept() will wait on first subflow sk_wq, and we always wakes up 106e1ff9e82SDavide Caratti * via msk->sk_socket 107e1ff9e82SDavide Caratti */ 108e1ff9e82SDavide Caratti RCU_INIT_POINTER(msk->first->sk_wq, &sk->sk_socket->wq); 109e1ff9e82SDavide Caratti 110fa68018dSPaolo Abeni return 0; 1112303f994SPeter Krystad } 1122303f994SPeter Krystad 113*8268ed4cSPaolo Abeni static bool mptcp_try_coalesce(struct sock *sk, struct sk_buff *to, 114*8268ed4cSPaolo Abeni struct sk_buff *from) 115*8268ed4cSPaolo Abeni { 116*8268ed4cSPaolo Abeni bool fragstolen; 117*8268ed4cSPaolo Abeni int delta; 118*8268ed4cSPaolo Abeni 119*8268ed4cSPaolo Abeni if (MPTCP_SKB_CB(from)->offset || 120*8268ed4cSPaolo Abeni !skb_try_coalesce(to, from, &fragstolen, &delta)) 121*8268ed4cSPaolo Abeni return false; 122*8268ed4cSPaolo Abeni 123*8268ed4cSPaolo Abeni kfree_skb_partial(from, fragstolen); 124*8268ed4cSPaolo Abeni atomic_add(delta, &sk->sk_rmem_alloc); 125*8268ed4cSPaolo Abeni sk_mem_charge(sk, delta); 126*8268ed4cSPaolo Abeni return true; 127*8268ed4cSPaolo Abeni } 128*8268ed4cSPaolo Abeni 1296771bfd9SFlorian Westphal static void __mptcp_move_skb(struct mptcp_sock *msk, struct sock *ssk, 1306771bfd9SFlorian Westphal struct sk_buff *skb, 1316771bfd9SFlorian Westphal unsigned int offset, size_t copy_len) 1326771bfd9SFlorian Westphal { 1336771bfd9SFlorian Westphal struct sock *sk = (struct sock *)msk; 1344e637c70SFlorian Westphal struct sk_buff *tail; 1356771bfd9SFlorian Westphal 1366771bfd9SFlorian Westphal __skb_unlink(skb, &ssk->sk_receive_queue); 1374e637c70SFlorian Westphal 1384e637c70SFlorian Westphal skb_ext_reset(skb); 1394e637c70SFlorian Westphal skb_orphan(skb); 140*8268ed4cSPaolo Abeni MPTCP_SKB_CB(skb)->offset = offset; 1414e637c70SFlorian Westphal msk->ack_seq += copy_len; 1424e637c70SFlorian Westphal 1434e637c70SFlorian Westphal tail = skb_peek_tail(&sk->sk_receive_queue); 144*8268ed4cSPaolo Abeni if (tail && mptcp_try_coalesce(sk, tail, skb)) 1454e637c70SFlorian Westphal return; 1464e637c70SFlorian Westphal 147600911ffSFlorian Westphal skb_set_owner_r(skb, sk); 1486771bfd9SFlorian Westphal __skb_queue_tail(&sk->sk_receive_queue, skb); 1496771bfd9SFlorian Westphal } 1506771bfd9SFlorian Westphal 15116a9a9daSMat Martineau static void mptcp_stop_timer(struct sock *sk) 15216a9a9daSMat Martineau { 15316a9a9daSMat Martineau struct inet_connection_sock *icsk = inet_csk(sk); 15416a9a9daSMat Martineau 15516a9a9daSMat Martineau sk_stop_timer(sk, &icsk->icsk_retransmit_timer); 15616a9a9daSMat Martineau mptcp_sk(sk)->timer_ival = 0; 15716a9a9daSMat Martineau } 15816a9a9daSMat Martineau 159de06f573SFlorian Westphal /* both sockets must be locked */ 160de06f573SFlorian Westphal static bool mptcp_subflow_dsn_valid(const struct mptcp_sock *msk, 161de06f573SFlorian Westphal struct sock *ssk) 162de06f573SFlorian Westphal { 163de06f573SFlorian Westphal struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk); 164de06f573SFlorian Westphal u64 dsn = mptcp_subflow_get_mapped_dsn(subflow); 165de06f573SFlorian Westphal 166de06f573SFlorian Westphal /* revalidate data sequence number. 167de06f573SFlorian Westphal * 168de06f573SFlorian Westphal * mptcp_subflow_data_available() is usually called 169de06f573SFlorian Westphal * without msk lock. Its unlikely (but possible) 170de06f573SFlorian Westphal * that msk->ack_seq has been advanced since the last 171de06f573SFlorian Westphal * call found in-sequence data. 172de06f573SFlorian Westphal */ 173de06f573SFlorian Westphal if (likely(dsn == msk->ack_seq)) 174de06f573SFlorian Westphal return true; 175de06f573SFlorian Westphal 176de06f573SFlorian Westphal subflow->data_avail = 0; 1776719331cSPaolo Abeni mptcp_subflow_data_available(ssk); 1786719331cSPaolo Abeni return subflow->data_avail == MPTCP_SUBFLOW_DATA_AVAIL; 179de06f573SFlorian Westphal } 180de06f573SFlorian Westphal 18116a9a9daSMat Martineau static void mptcp_check_data_fin_ack(struct sock *sk) 18216a9a9daSMat Martineau { 18316a9a9daSMat Martineau struct mptcp_sock *msk = mptcp_sk(sk); 18416a9a9daSMat Martineau 18516a9a9daSMat Martineau if (__mptcp_check_fallback(msk)) 18616a9a9daSMat Martineau return; 18716a9a9daSMat Martineau 18816a9a9daSMat Martineau /* Look for an acknowledged DATA_FIN */ 18916a9a9daSMat Martineau if (((1 << sk->sk_state) & 19016a9a9daSMat Martineau (TCPF_FIN_WAIT1 | TCPF_CLOSING | TCPF_LAST_ACK)) && 19116a9a9daSMat Martineau msk->write_seq == atomic64_read(&msk->snd_una)) { 19216a9a9daSMat Martineau mptcp_stop_timer(sk); 19316a9a9daSMat Martineau 19416a9a9daSMat Martineau WRITE_ONCE(msk->snd_data_fin_enable, 0); 19516a9a9daSMat Martineau 19616a9a9daSMat Martineau switch (sk->sk_state) { 19716a9a9daSMat Martineau case TCP_FIN_WAIT1: 19816a9a9daSMat Martineau inet_sk_state_store(sk, TCP_FIN_WAIT2); 19916a9a9daSMat Martineau sk->sk_state_change(sk); 20016a9a9daSMat Martineau break; 20116a9a9daSMat Martineau case TCP_CLOSING: 20216a9a9daSMat Martineau case TCP_LAST_ACK: 20316a9a9daSMat Martineau inet_sk_state_store(sk, TCP_CLOSE); 20416a9a9daSMat Martineau sk->sk_state_change(sk); 20516a9a9daSMat Martineau break; 20616a9a9daSMat Martineau } 20716a9a9daSMat Martineau 20816a9a9daSMat Martineau if (sk->sk_shutdown == SHUTDOWN_MASK || 20916a9a9daSMat Martineau sk->sk_state == TCP_CLOSE) 21016a9a9daSMat Martineau sk_wake_async(sk, SOCK_WAKE_WAITD, POLL_HUP); 21116a9a9daSMat Martineau else 21216a9a9daSMat Martineau sk_wake_async(sk, SOCK_WAKE_WAITD, POLL_IN); 21316a9a9daSMat Martineau } 21416a9a9daSMat Martineau } 21516a9a9daSMat Martineau 2163721b9b6SMat Martineau static bool mptcp_pending_data_fin(struct sock *sk, u64 *seq) 2173721b9b6SMat Martineau { 2183721b9b6SMat Martineau struct mptcp_sock *msk = mptcp_sk(sk); 2193721b9b6SMat Martineau 2203721b9b6SMat Martineau if (READ_ONCE(msk->rcv_data_fin) && 2213721b9b6SMat Martineau ((1 << sk->sk_state) & 2223721b9b6SMat Martineau (TCPF_ESTABLISHED | TCPF_FIN_WAIT1 | TCPF_FIN_WAIT2))) { 2233721b9b6SMat Martineau u64 rcv_data_fin_seq = READ_ONCE(msk->rcv_data_fin_seq); 2243721b9b6SMat Martineau 2253721b9b6SMat Martineau if (msk->ack_seq == rcv_data_fin_seq) { 2263721b9b6SMat Martineau if (seq) 2273721b9b6SMat Martineau *seq = rcv_data_fin_seq; 2283721b9b6SMat Martineau 2293721b9b6SMat Martineau return true; 2303721b9b6SMat Martineau } 2313721b9b6SMat Martineau } 2323721b9b6SMat Martineau 2333721b9b6SMat Martineau return false; 2343721b9b6SMat Martineau } 2353721b9b6SMat Martineau 2363721b9b6SMat Martineau static void mptcp_set_timeout(const struct sock *sk, const struct sock *ssk) 2373721b9b6SMat Martineau { 2383721b9b6SMat Martineau long tout = ssk && inet_csk(ssk)->icsk_pending ? 2393721b9b6SMat Martineau inet_csk(ssk)->icsk_timeout - jiffies : 0; 2403721b9b6SMat Martineau 2413721b9b6SMat Martineau if (tout <= 0) 2423721b9b6SMat Martineau tout = mptcp_sk(sk)->timer_ival; 2433721b9b6SMat Martineau mptcp_sk(sk)->timer_ival = tout > 0 ? tout : TCP_RTO_MIN; 2443721b9b6SMat Martineau } 2453721b9b6SMat Martineau 2463721b9b6SMat Martineau static void mptcp_check_data_fin(struct sock *sk) 2473721b9b6SMat Martineau { 2483721b9b6SMat Martineau struct mptcp_sock *msk = mptcp_sk(sk); 2493721b9b6SMat Martineau u64 rcv_data_fin_seq; 2503721b9b6SMat Martineau 2513721b9b6SMat Martineau if (__mptcp_check_fallback(msk) || !msk->first) 2523721b9b6SMat Martineau return; 2533721b9b6SMat Martineau 2543721b9b6SMat Martineau /* Need to ack a DATA_FIN received from a peer while this side 2553721b9b6SMat Martineau * of the connection is in ESTABLISHED, FIN_WAIT1, or FIN_WAIT2. 2563721b9b6SMat Martineau * msk->rcv_data_fin was set when parsing the incoming options 2573721b9b6SMat Martineau * at the subflow level and the msk lock was not held, so this 2583721b9b6SMat Martineau * is the first opportunity to act on the DATA_FIN and change 2593721b9b6SMat Martineau * the msk state. 2603721b9b6SMat Martineau * 2613721b9b6SMat Martineau * If we are caught up to the sequence number of the incoming 2623721b9b6SMat Martineau * DATA_FIN, send the DATA_ACK now and do state transition. If 2633721b9b6SMat Martineau * not caught up, do nothing and let the recv code send DATA_ACK 2643721b9b6SMat Martineau * when catching up. 2653721b9b6SMat Martineau */ 2663721b9b6SMat Martineau 2673721b9b6SMat Martineau if (mptcp_pending_data_fin(sk, &rcv_data_fin_seq)) { 2683721b9b6SMat Martineau struct mptcp_subflow_context *subflow; 2693721b9b6SMat Martineau 2703721b9b6SMat Martineau msk->ack_seq++; 2713721b9b6SMat Martineau WRITE_ONCE(msk->rcv_data_fin, 0); 2723721b9b6SMat Martineau 2733721b9b6SMat Martineau sk->sk_shutdown |= RCV_SHUTDOWN; 27416a9a9daSMat Martineau smp_mb__before_atomic(); /* SHUTDOWN must be visible first */ 27516a9a9daSMat Martineau set_bit(MPTCP_DATA_READY, &msk->flags); 2763721b9b6SMat Martineau 2773721b9b6SMat Martineau switch (sk->sk_state) { 2783721b9b6SMat Martineau case TCP_ESTABLISHED: 2793721b9b6SMat Martineau inet_sk_state_store(sk, TCP_CLOSE_WAIT); 2803721b9b6SMat Martineau break; 2813721b9b6SMat Martineau case TCP_FIN_WAIT1: 2823721b9b6SMat Martineau inet_sk_state_store(sk, TCP_CLOSING); 2833721b9b6SMat Martineau break; 2843721b9b6SMat Martineau case TCP_FIN_WAIT2: 2853721b9b6SMat Martineau inet_sk_state_store(sk, TCP_CLOSE); 2863721b9b6SMat Martineau // @@ Close subflows now? 2873721b9b6SMat Martineau break; 2883721b9b6SMat Martineau default: 2893721b9b6SMat Martineau /* Other states not expected */ 2903721b9b6SMat Martineau WARN_ON_ONCE(1); 2913721b9b6SMat Martineau break; 2923721b9b6SMat Martineau } 2933721b9b6SMat Martineau 2943721b9b6SMat Martineau mptcp_set_timeout(sk, NULL); 2953721b9b6SMat Martineau mptcp_for_each_subflow(msk, subflow) { 2963721b9b6SMat Martineau struct sock *ssk = mptcp_subflow_tcp_sock(subflow); 2973721b9b6SMat Martineau 2983721b9b6SMat Martineau lock_sock(ssk); 2993721b9b6SMat Martineau tcp_send_ack(ssk); 3003721b9b6SMat Martineau release_sock(ssk); 3013721b9b6SMat Martineau } 3023721b9b6SMat Martineau 3033721b9b6SMat Martineau sk->sk_state_change(sk); 3043721b9b6SMat Martineau 3053721b9b6SMat Martineau if (sk->sk_shutdown == SHUTDOWN_MASK || 3063721b9b6SMat Martineau sk->sk_state == TCP_CLOSE) 3073721b9b6SMat Martineau sk_wake_async(sk, SOCK_WAKE_WAITD, POLL_HUP); 3083721b9b6SMat Martineau else 3093721b9b6SMat Martineau sk_wake_async(sk, SOCK_WAKE_WAITD, POLL_IN); 3103721b9b6SMat Martineau } 3113721b9b6SMat Martineau } 3123721b9b6SMat Martineau 3136771bfd9SFlorian Westphal static bool __mptcp_move_skbs_from_subflow(struct mptcp_sock *msk, 3146771bfd9SFlorian Westphal struct sock *ssk, 3156771bfd9SFlorian Westphal unsigned int *bytes) 3166771bfd9SFlorian Westphal { 3176771bfd9SFlorian Westphal struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk); 318600911ffSFlorian Westphal struct sock *sk = (struct sock *)msk; 3196771bfd9SFlorian Westphal unsigned int moved = 0; 3206771bfd9SFlorian Westphal bool more_data_avail; 3216771bfd9SFlorian Westphal struct tcp_sock *tp; 3226771bfd9SFlorian Westphal bool done = false; 323600911ffSFlorian Westphal 3246719331cSPaolo Abeni pr_debug("msk=%p ssk=%p data avail=%d valid=%d empty=%d", 3256719331cSPaolo Abeni msk, ssk, subflow->data_avail, 3266719331cSPaolo Abeni mptcp_subflow_dsn_valid(msk, ssk), 3276719331cSPaolo Abeni !skb_peek(&ssk->sk_receive_queue)); 3286719331cSPaolo Abeni if (subflow->data_avail == MPTCP_SUBFLOW_OOO_DATA) { 3296719331cSPaolo Abeni mptcp_subflow_discard_data(ssk, subflow->map_data_len); 330de06f573SFlorian Westphal return false; 331de06f573SFlorian Westphal } 332de06f573SFlorian Westphal 3336719331cSPaolo Abeni if (!mptcp_subflow_dsn_valid(msk, ssk)) 3346719331cSPaolo Abeni return false; 3356719331cSPaolo Abeni 3366771bfd9SFlorian Westphal tp = tcp_sk(ssk); 3376771bfd9SFlorian Westphal do { 3386771bfd9SFlorian Westphal u32 map_remaining, offset; 3396771bfd9SFlorian Westphal u32 seq = tp->copied_seq; 3406771bfd9SFlorian Westphal struct sk_buff *skb; 3416771bfd9SFlorian Westphal bool fin; 3426771bfd9SFlorian Westphal 3436771bfd9SFlorian Westphal /* try to move as much data as available */ 3446771bfd9SFlorian Westphal map_remaining = subflow->map_data_len - 3456771bfd9SFlorian Westphal mptcp_subflow_get_map_offset(subflow); 3466771bfd9SFlorian Westphal 3476771bfd9SFlorian Westphal skb = skb_peek(&ssk->sk_receive_queue); 3486771bfd9SFlorian Westphal if (!skb) 3496771bfd9SFlorian Westphal break; 3506771bfd9SFlorian Westphal 351e1ff9e82SDavide Caratti if (__mptcp_check_fallback(msk)) { 352e1ff9e82SDavide Caratti /* if we are running under the workqueue, TCP could have 353e1ff9e82SDavide Caratti * collapsed skbs between dummy map creation and now 354e1ff9e82SDavide Caratti * be sure to adjust the size 355e1ff9e82SDavide Caratti */ 356e1ff9e82SDavide Caratti map_remaining = skb->len; 357e1ff9e82SDavide Caratti subflow->map_data_len = skb->len; 358e1ff9e82SDavide Caratti } 359e1ff9e82SDavide Caratti 3606771bfd9SFlorian Westphal offset = seq - TCP_SKB_CB(skb)->seq; 3616771bfd9SFlorian Westphal fin = TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN; 3626771bfd9SFlorian Westphal if (fin) { 3636771bfd9SFlorian Westphal done = true; 3646771bfd9SFlorian Westphal seq++; 3656771bfd9SFlorian Westphal } 3666771bfd9SFlorian Westphal 3676771bfd9SFlorian Westphal if (offset < skb->len) { 3686771bfd9SFlorian Westphal size_t len = skb->len - offset; 3696771bfd9SFlorian Westphal 3706771bfd9SFlorian Westphal if (tp->urg_data) 3716771bfd9SFlorian Westphal done = true; 3726771bfd9SFlorian Westphal 3736771bfd9SFlorian Westphal __mptcp_move_skb(msk, ssk, skb, offset, len); 3746771bfd9SFlorian Westphal seq += len; 3756771bfd9SFlorian Westphal moved += len; 3766771bfd9SFlorian Westphal 3776771bfd9SFlorian Westphal if (WARN_ON_ONCE(map_remaining < len)) 3786771bfd9SFlorian Westphal break; 3796771bfd9SFlorian Westphal } else { 3806771bfd9SFlorian Westphal WARN_ON_ONCE(!fin); 3816771bfd9SFlorian Westphal sk_eat_skb(ssk, skb); 3826771bfd9SFlorian Westphal done = true; 3836771bfd9SFlorian Westphal } 3846771bfd9SFlorian Westphal 3856771bfd9SFlorian Westphal WRITE_ONCE(tp->copied_seq, seq); 3866771bfd9SFlorian Westphal more_data_avail = mptcp_subflow_data_available(ssk); 387600911ffSFlorian Westphal 388600911ffSFlorian Westphal if (atomic_read(&sk->sk_rmem_alloc) > READ_ONCE(sk->sk_rcvbuf)) { 389600911ffSFlorian Westphal done = true; 390600911ffSFlorian Westphal break; 391600911ffSFlorian Westphal } 3926771bfd9SFlorian Westphal } while (more_data_avail); 3936771bfd9SFlorian Westphal 3946719331cSPaolo Abeni *bytes += moved; 3956771bfd9SFlorian Westphal 39643b54c6eSMat Martineau /* If the moves have caught up with the DATA_FIN sequence number 39743b54c6eSMat Martineau * it's time to ack the DATA_FIN and change socket state, but 39843b54c6eSMat Martineau * this is not a good place to change state. Let the workqueue 39943b54c6eSMat Martineau * do it. 40043b54c6eSMat Martineau */ 40143b54c6eSMat Martineau if (mptcp_pending_data_fin(sk, NULL) && 40243b54c6eSMat Martineau schedule_work(&msk->work)) 40343b54c6eSMat Martineau sock_hold(sk); 40443b54c6eSMat Martineau 4056771bfd9SFlorian Westphal return done; 4066771bfd9SFlorian Westphal } 4076771bfd9SFlorian Westphal 4082e52213cSFlorian Westphal /* In most cases we will be able to lock the mptcp socket. If its already 4092e52213cSFlorian Westphal * owned, we need to defer to the work queue to avoid ABBA deadlock. 4102e52213cSFlorian Westphal */ 4112e52213cSFlorian Westphal static bool move_skbs_to_msk(struct mptcp_sock *msk, struct sock *ssk) 4122e52213cSFlorian Westphal { 4132e52213cSFlorian Westphal struct sock *sk = (struct sock *)msk; 4142e52213cSFlorian Westphal unsigned int moved = 0; 4152e52213cSFlorian Westphal 4162e52213cSFlorian Westphal if (READ_ONCE(sk->sk_lock.owned)) 4172e52213cSFlorian Westphal return false; 4182e52213cSFlorian Westphal 4192e52213cSFlorian Westphal if (unlikely(!spin_trylock_bh(&sk->sk_lock.slock))) 4202e52213cSFlorian Westphal return false; 4212e52213cSFlorian Westphal 4222e52213cSFlorian Westphal /* must re-check after taking the lock */ 4232e52213cSFlorian Westphal if (!READ_ONCE(sk->sk_lock.owned)) 4242e52213cSFlorian Westphal __mptcp_move_skbs_from_subflow(msk, ssk, &moved); 4252e52213cSFlorian Westphal 4262e52213cSFlorian Westphal spin_unlock_bh(&sk->sk_lock.slock); 4272e52213cSFlorian Westphal 4282e52213cSFlorian Westphal return moved > 0; 4292e52213cSFlorian Westphal } 4302e52213cSFlorian Westphal 4312e52213cSFlorian Westphal void mptcp_data_ready(struct sock *sk, struct sock *ssk) 432101f6f85SFlorian Westphal { 4336719331cSPaolo Abeni struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk); 434101f6f85SFlorian Westphal struct mptcp_sock *msk = mptcp_sk(sk); 4356719331cSPaolo Abeni bool wake; 436101f6f85SFlorian Westphal 4376719331cSPaolo Abeni /* move_skbs_to_msk below can legitly clear the data_avail flag, 4386719331cSPaolo Abeni * but we will need later to properly woke the reader, cache its 4396719331cSPaolo Abeni * value 4406719331cSPaolo Abeni */ 4416719331cSPaolo Abeni wake = subflow->data_avail == MPTCP_SUBFLOW_DATA_AVAIL; 4426719331cSPaolo Abeni if (wake) 443101f6f85SFlorian Westphal set_bit(MPTCP_DATA_READY, &msk->flags); 4446771bfd9SFlorian Westphal 4452e52213cSFlorian Westphal if (atomic_read(&sk->sk_rmem_alloc) < READ_ONCE(sk->sk_rcvbuf) && 4462e52213cSFlorian Westphal move_skbs_to_msk(msk, ssk)) 4472e52213cSFlorian Westphal goto wake; 4482e52213cSFlorian Westphal 449600911ffSFlorian Westphal /* don't schedule if mptcp sk is (still) over limit */ 450600911ffSFlorian Westphal if (atomic_read(&sk->sk_rmem_alloc) > READ_ONCE(sk->sk_rcvbuf)) 451600911ffSFlorian Westphal goto wake; 452600911ffSFlorian Westphal 45314c441b5SPaolo Abeni /* mptcp socket is owned, release_cb should retry */ 45414c441b5SPaolo Abeni if (!test_and_set_bit(TCP_DELACK_TIMER_DEFERRED, 45514c441b5SPaolo Abeni &sk->sk_tsq_flags)) { 45614c441b5SPaolo Abeni sock_hold(sk); 4576771bfd9SFlorian Westphal 45814c441b5SPaolo Abeni /* need to try again, its possible release_cb() has already 45914c441b5SPaolo Abeni * been called after the test_and_set_bit() above. 46014c441b5SPaolo Abeni */ 46114c441b5SPaolo Abeni move_skbs_to_msk(msk, ssk); 46214c441b5SPaolo Abeni } 463600911ffSFlorian Westphal wake: 4646719331cSPaolo Abeni if (wake) 465101f6f85SFlorian Westphal sk->sk_data_ready(sk); 466101f6f85SFlorian Westphal } 467101f6f85SFlorian Westphal 468ec3edaa7SPeter Krystad static void __mptcp_flush_join_list(struct mptcp_sock *msk) 469ec3edaa7SPeter Krystad { 470ec3edaa7SPeter Krystad if (likely(list_empty(&msk->join_list))) 471ec3edaa7SPeter Krystad return; 472ec3edaa7SPeter Krystad 473ec3edaa7SPeter Krystad spin_lock_bh(&msk->join_list_lock); 474ec3edaa7SPeter Krystad list_splice_tail_init(&msk->join_list, &msk->conn_list); 475ec3edaa7SPeter Krystad spin_unlock_bh(&msk->join_list_lock); 476ec3edaa7SPeter Krystad } 477ec3edaa7SPeter Krystad 478b51f9b80SPaolo Abeni static bool mptcp_timer_pending(struct sock *sk) 479b51f9b80SPaolo Abeni { 480b51f9b80SPaolo Abeni return timer_pending(&inet_csk(sk)->icsk_retransmit_timer); 481b51f9b80SPaolo Abeni } 482b51f9b80SPaolo Abeni 483b51f9b80SPaolo Abeni static void mptcp_reset_timer(struct sock *sk) 484b51f9b80SPaolo Abeni { 485b51f9b80SPaolo Abeni struct inet_connection_sock *icsk = inet_csk(sk); 486b51f9b80SPaolo Abeni unsigned long tout; 487b51f9b80SPaolo Abeni 488b51f9b80SPaolo Abeni /* should never be called with mptcp level timer cleared */ 489b51f9b80SPaolo Abeni tout = READ_ONCE(mptcp_sk(sk)->timer_ival); 490b51f9b80SPaolo Abeni if (WARN_ON_ONCE(!tout)) 491b51f9b80SPaolo Abeni tout = TCP_RTO_MIN; 492b51f9b80SPaolo Abeni sk_reset_timer(sk, &icsk->icsk_retransmit_timer, jiffies + tout); 493b51f9b80SPaolo Abeni } 494b51f9b80SPaolo Abeni 495b51f9b80SPaolo Abeni void mptcp_data_acked(struct sock *sk) 496b51f9b80SPaolo Abeni { 497b51f9b80SPaolo Abeni mptcp_reset_timer(sk); 4983b1d6210SPaolo Abeni 49963561a40SPaolo Abeni if ((!test_bit(MPTCP_SEND_SPACE, &mptcp_sk(sk)->flags) || 50043b54c6eSMat Martineau (inet_sk_state_load(sk) != TCP_ESTABLISHED)) && 5013b1d6210SPaolo Abeni schedule_work(&mptcp_sk(sk)->work)) 5023b1d6210SPaolo Abeni sock_hold(sk); 503b51f9b80SPaolo Abeni } 504b51f9b80SPaolo Abeni 50559832e24SFlorian Westphal void mptcp_subflow_eof(struct sock *sk) 50659832e24SFlorian Westphal { 50759832e24SFlorian Westphal struct mptcp_sock *msk = mptcp_sk(sk); 50859832e24SFlorian Westphal 50959832e24SFlorian Westphal if (!test_and_set_bit(MPTCP_WORK_EOF, &msk->flags) && 51059832e24SFlorian Westphal schedule_work(&msk->work)) 51159832e24SFlorian Westphal sock_hold(sk); 51259832e24SFlorian Westphal } 51359832e24SFlorian Westphal 5145969856aSPaolo Abeni static void mptcp_check_for_eof(struct mptcp_sock *msk) 5155969856aSPaolo Abeni { 5165969856aSPaolo Abeni struct mptcp_subflow_context *subflow; 5175969856aSPaolo Abeni struct sock *sk = (struct sock *)msk; 5185969856aSPaolo Abeni int receivers = 0; 5195969856aSPaolo Abeni 5205969856aSPaolo Abeni mptcp_for_each_subflow(msk, subflow) 5215969856aSPaolo Abeni receivers += !subflow->rx_eof; 5225969856aSPaolo Abeni 5235969856aSPaolo Abeni if (!receivers && !(sk->sk_shutdown & RCV_SHUTDOWN)) { 5245969856aSPaolo Abeni /* hopefully temporary hack: propagate shutdown status 5255969856aSPaolo Abeni * to msk, when all subflows agree on it 5265969856aSPaolo Abeni */ 5275969856aSPaolo Abeni sk->sk_shutdown |= RCV_SHUTDOWN; 5285969856aSPaolo Abeni 5295969856aSPaolo Abeni smp_mb__before_atomic(); /* SHUTDOWN must be visible first */ 5305969856aSPaolo Abeni set_bit(MPTCP_DATA_READY, &msk->flags); 5315969856aSPaolo Abeni sk->sk_data_ready(sk); 5325969856aSPaolo Abeni } 5335969856aSPaolo Abeni } 5345969856aSPaolo Abeni 5356d0060f6SMat Martineau static bool mptcp_ext_cache_refill(struct mptcp_sock *msk) 5366d0060f6SMat Martineau { 5374930f483SFlorian Westphal const struct sock *sk = (const struct sock *)msk; 5384930f483SFlorian Westphal 5396d0060f6SMat Martineau if (!msk->cached_ext) 5404930f483SFlorian Westphal msk->cached_ext = __skb_ext_alloc(sk->sk_allocation); 5416d0060f6SMat Martineau 5426d0060f6SMat Martineau return !!msk->cached_ext; 5436d0060f6SMat Martineau } 5446d0060f6SMat Martineau 5457a6a6cbcSPaolo Abeni static struct sock *mptcp_subflow_recv_lookup(const struct mptcp_sock *msk) 5467a6a6cbcSPaolo Abeni { 5477a6a6cbcSPaolo Abeni struct mptcp_subflow_context *subflow; 5487a6a6cbcSPaolo Abeni struct sock *sk = (struct sock *)msk; 5497a6a6cbcSPaolo Abeni 5507a6a6cbcSPaolo Abeni sock_owned_by_me(sk); 5517a6a6cbcSPaolo Abeni 5527a6a6cbcSPaolo Abeni mptcp_for_each_subflow(msk, subflow) { 5537a6a6cbcSPaolo Abeni if (subflow->data_avail) 5547a6a6cbcSPaolo Abeni return mptcp_subflow_tcp_sock(subflow); 5557a6a6cbcSPaolo Abeni } 5567a6a6cbcSPaolo Abeni 5577a6a6cbcSPaolo Abeni return NULL; 5587a6a6cbcSPaolo Abeni } 5597a6a6cbcSPaolo Abeni 5603f8e0aaeSPaolo Abeni static bool mptcp_skb_can_collapse_to(u64 write_seq, 56157040755SPaolo Abeni const struct sk_buff *skb, 56257040755SPaolo Abeni const struct mptcp_ext *mpext) 5636d0060f6SMat Martineau { 56457040755SPaolo Abeni if (!tcp_skb_can_collapse_to(skb)) 56557040755SPaolo Abeni return false; 56657040755SPaolo Abeni 56757040755SPaolo Abeni /* can collapse only if MPTCP level sequence is in order */ 5683f8e0aaeSPaolo Abeni return mpext && mpext->data_seq + mpext->data_len == write_seq; 56957040755SPaolo Abeni } 57057040755SPaolo Abeni 57118b683bfSPaolo Abeni static bool mptcp_frag_can_collapse_to(const struct mptcp_sock *msk, 57218b683bfSPaolo Abeni const struct page_frag *pfrag, 57318b683bfSPaolo Abeni const struct mptcp_data_frag *df) 57418b683bfSPaolo Abeni { 57518b683bfSPaolo Abeni return df && pfrag->page == df->page && 57618b683bfSPaolo Abeni df->data_seq + df->data_len == msk->write_seq; 57718b683bfSPaolo Abeni } 57818b683bfSPaolo Abeni 579d027236cSPaolo Abeni static void dfrag_uncharge(struct sock *sk, int len) 58018b683bfSPaolo Abeni { 581d027236cSPaolo Abeni sk_mem_uncharge(sk, len); 5827948f6ccSFlorian Westphal sk_wmem_queued_add(sk, -len); 583d027236cSPaolo Abeni } 584d027236cSPaolo Abeni 585d027236cSPaolo Abeni static void dfrag_clear(struct sock *sk, struct mptcp_data_frag *dfrag) 586d027236cSPaolo Abeni { 587d027236cSPaolo Abeni int len = dfrag->data_len + dfrag->overhead; 588d027236cSPaolo Abeni 58918b683bfSPaolo Abeni list_del(&dfrag->list); 590d027236cSPaolo Abeni dfrag_uncharge(sk, len); 59118b683bfSPaolo Abeni put_page(dfrag->page); 59218b683bfSPaolo Abeni } 59318b683bfSPaolo Abeni 59463561a40SPaolo Abeni static bool mptcp_is_writeable(struct mptcp_sock *msk) 59563561a40SPaolo Abeni { 59663561a40SPaolo Abeni struct mptcp_subflow_context *subflow; 59763561a40SPaolo Abeni 59863561a40SPaolo Abeni if (!sk_stream_is_writeable((struct sock *)msk)) 59963561a40SPaolo Abeni return false; 60063561a40SPaolo Abeni 60163561a40SPaolo Abeni mptcp_for_each_subflow(msk, subflow) { 60263561a40SPaolo Abeni if (sk_stream_is_writeable(subflow->tcp_sock)) 60363561a40SPaolo Abeni return true; 60463561a40SPaolo Abeni } 60563561a40SPaolo Abeni return false; 60663561a40SPaolo Abeni } 60763561a40SPaolo Abeni 60818b683bfSPaolo Abeni static void mptcp_clean_una(struct sock *sk) 60918b683bfSPaolo Abeni { 61018b683bfSPaolo Abeni struct mptcp_sock *msk = mptcp_sk(sk); 61118b683bfSPaolo Abeni struct mptcp_data_frag *dtmp, *dfrag; 612d027236cSPaolo Abeni bool cleaned = false; 613e1ff9e82SDavide Caratti u64 snd_una; 614e1ff9e82SDavide Caratti 615e1ff9e82SDavide Caratti /* on fallback we just need to ignore snd_una, as this is really 616e1ff9e82SDavide Caratti * plain TCP 617e1ff9e82SDavide Caratti */ 618e1ff9e82SDavide Caratti if (__mptcp_check_fallback(msk)) 619e1ff9e82SDavide Caratti atomic64_set(&msk->snd_una, msk->write_seq); 620e1ff9e82SDavide Caratti snd_una = atomic64_read(&msk->snd_una); 62118b683bfSPaolo Abeni 62218b683bfSPaolo Abeni list_for_each_entry_safe(dfrag, dtmp, &msk->rtx_queue, list) { 62318b683bfSPaolo Abeni if (after64(dfrag->data_seq + dfrag->data_len, snd_una)) 62418b683bfSPaolo Abeni break; 62518b683bfSPaolo Abeni 626d027236cSPaolo Abeni dfrag_clear(sk, dfrag); 627d027236cSPaolo Abeni cleaned = true; 628d027236cSPaolo Abeni } 629d027236cSPaolo Abeni 6307948f6ccSFlorian Westphal dfrag = mptcp_rtx_head(sk); 6317948f6ccSFlorian Westphal if (dfrag && after64(snd_una, dfrag->data_seq)) { 63253eb4c38SPaolo Abeni u64 delta = snd_una - dfrag->data_seq; 63353eb4c38SPaolo Abeni 63453eb4c38SPaolo Abeni if (WARN_ON_ONCE(delta > dfrag->data_len)) 63553eb4c38SPaolo Abeni goto out; 6367948f6ccSFlorian Westphal 6377948f6ccSFlorian Westphal dfrag->data_seq += delta; 63853eb4c38SPaolo Abeni dfrag->offset += delta; 6397948f6ccSFlorian Westphal dfrag->data_len -= delta; 6407948f6ccSFlorian Westphal 6417948f6ccSFlorian Westphal dfrag_uncharge(sk, delta); 6427948f6ccSFlorian Westphal cleaned = true; 6437948f6ccSFlorian Westphal } 6447948f6ccSFlorian Westphal 64553eb4c38SPaolo Abeni out: 646d027236cSPaolo Abeni if (cleaned) { 647d027236cSPaolo Abeni sk_mem_reclaim_partial(sk); 6487948f6ccSFlorian Westphal 6497948f6ccSFlorian Westphal /* Only wake up writers if a subflow is ready */ 65063561a40SPaolo Abeni if (mptcp_is_writeable(msk)) { 65163561a40SPaolo Abeni set_bit(MPTCP_SEND_SPACE, &mptcp_sk(sk)->flags); 65263561a40SPaolo Abeni smp_mb__after_atomic(); 65363561a40SPaolo Abeni 65463561a40SPaolo Abeni /* set SEND_SPACE before sk_stream_write_space clears 65563561a40SPaolo Abeni * NOSPACE 65663561a40SPaolo Abeni */ 6577948f6ccSFlorian Westphal sk_stream_write_space(sk); 65818b683bfSPaolo Abeni } 65918b683bfSPaolo Abeni } 66063561a40SPaolo Abeni } 66118b683bfSPaolo Abeni 66218b683bfSPaolo Abeni /* ensure we get enough memory for the frag hdr, beyond some minimal amount of 66318b683bfSPaolo Abeni * data 66418b683bfSPaolo Abeni */ 66518b683bfSPaolo Abeni static bool mptcp_page_frag_refill(struct sock *sk, struct page_frag *pfrag) 66618b683bfSPaolo Abeni { 66718b683bfSPaolo Abeni if (likely(skb_page_frag_refill(32U + sizeof(struct mptcp_data_frag), 66818b683bfSPaolo Abeni pfrag, sk->sk_allocation))) 66918b683bfSPaolo Abeni return true; 67018b683bfSPaolo Abeni 67118b683bfSPaolo Abeni sk->sk_prot->enter_memory_pressure(sk); 67218b683bfSPaolo Abeni sk_stream_moderate_sndbuf(sk); 67318b683bfSPaolo Abeni return false; 67418b683bfSPaolo Abeni } 67518b683bfSPaolo Abeni 67618b683bfSPaolo Abeni static struct mptcp_data_frag * 67718b683bfSPaolo Abeni mptcp_carve_data_frag(const struct mptcp_sock *msk, struct page_frag *pfrag, 67818b683bfSPaolo Abeni int orig_offset) 67918b683bfSPaolo Abeni { 68018b683bfSPaolo Abeni int offset = ALIGN(orig_offset, sizeof(long)); 68118b683bfSPaolo Abeni struct mptcp_data_frag *dfrag; 68218b683bfSPaolo Abeni 68318b683bfSPaolo Abeni dfrag = (struct mptcp_data_frag *)(page_to_virt(pfrag->page) + offset); 68418b683bfSPaolo Abeni dfrag->data_len = 0; 68518b683bfSPaolo Abeni dfrag->data_seq = msk->write_seq; 68618b683bfSPaolo Abeni dfrag->overhead = offset - orig_offset + sizeof(struct mptcp_data_frag); 68718b683bfSPaolo Abeni dfrag->offset = offset + sizeof(struct mptcp_data_frag); 68818b683bfSPaolo Abeni dfrag->page = pfrag->page; 68918b683bfSPaolo Abeni 69018b683bfSPaolo Abeni return dfrag; 69118b683bfSPaolo Abeni } 69218b683bfSPaolo Abeni 69357040755SPaolo Abeni static int mptcp_sendmsg_frag(struct sock *sk, struct sock *ssk, 6943f8e0aaeSPaolo Abeni struct msghdr *msg, struct mptcp_data_frag *dfrag, 6953f8e0aaeSPaolo Abeni long *timeo, int *pmss_now, 69657040755SPaolo Abeni int *ps_goal) 69757040755SPaolo Abeni { 69818b683bfSPaolo Abeni int mss_now, avail_size, size_goal, offset, ret, frag_truesize = 0; 69918b683bfSPaolo Abeni bool dfrag_collapsed, can_collapse = false; 7006d0060f6SMat Martineau struct mptcp_sock *msk = mptcp_sk(sk); 7016d0060f6SMat Martineau struct mptcp_ext *mpext = NULL; 7023f8e0aaeSPaolo Abeni bool retransmission = !!dfrag; 70357040755SPaolo Abeni struct sk_buff *skb, *tail; 7046d0060f6SMat Martineau struct page_frag *pfrag; 7053f8e0aaeSPaolo Abeni struct page *page; 7063f8e0aaeSPaolo Abeni u64 *write_seq; 7076d0060f6SMat Martineau size_t psize; 7086d0060f6SMat Martineau 7096d0060f6SMat Martineau /* use the mptcp page cache so that we can easily move the data 7106d0060f6SMat Martineau * from one substream to another, but do per subflow memory accounting 7113f8e0aaeSPaolo Abeni * Note: pfrag is used only !retransmission, but the compiler if 7123f8e0aaeSPaolo Abeni * fooled into a warning if we don't init here 7136d0060f6SMat Martineau */ 7146d0060f6SMat Martineau pfrag = sk_page_frag(sk); 7153f8e0aaeSPaolo Abeni if (!retransmission) { 7163f8e0aaeSPaolo Abeni write_seq = &msk->write_seq; 7173f8e0aaeSPaolo Abeni page = pfrag->page; 7183f8e0aaeSPaolo Abeni } else { 7193f8e0aaeSPaolo Abeni write_seq = &dfrag->data_seq; 7203f8e0aaeSPaolo Abeni page = dfrag->page; 7213f8e0aaeSPaolo Abeni } 7226d0060f6SMat Martineau 7236d0060f6SMat Martineau /* compute copy limit */ 7246d0060f6SMat Martineau mss_now = tcp_send_mss(ssk, &size_goal, msg->msg_flags); 72557040755SPaolo Abeni *pmss_now = mss_now; 72657040755SPaolo Abeni *ps_goal = size_goal; 72757040755SPaolo Abeni avail_size = size_goal; 72857040755SPaolo Abeni skb = tcp_write_queue_tail(ssk); 72957040755SPaolo Abeni if (skb) { 73057040755SPaolo Abeni mpext = skb_ext_find(skb, SKB_EXT_MPTCP); 7316d0060f6SMat Martineau 73257040755SPaolo Abeni /* Limit the write to the size available in the 73357040755SPaolo Abeni * current skb, if any, so that we create at most a new skb. 73457040755SPaolo Abeni * Explicitly tells TCP internals to avoid collapsing on later 73557040755SPaolo Abeni * queue management operation, to avoid breaking the ext <-> 73657040755SPaolo Abeni * SSN association set here 73757040755SPaolo Abeni */ 73857040755SPaolo Abeni can_collapse = (size_goal - skb->len > 0) && 7393f8e0aaeSPaolo Abeni mptcp_skb_can_collapse_to(*write_seq, skb, mpext); 74057040755SPaolo Abeni if (!can_collapse) 74157040755SPaolo Abeni TCP_SKB_CB(skb)->eor = 1; 74257040755SPaolo Abeni else 74357040755SPaolo Abeni avail_size = size_goal - skb->len; 74457040755SPaolo Abeni } 74518b683bfSPaolo Abeni 7463f8e0aaeSPaolo Abeni if (!retransmission) { 7473f8e0aaeSPaolo Abeni /* reuse tail pfrag, if possible, or carve a new one from the 7483f8e0aaeSPaolo Abeni * page allocator 74918b683bfSPaolo Abeni */ 75018b683bfSPaolo Abeni dfrag = mptcp_rtx_tail(sk); 75118b683bfSPaolo Abeni offset = pfrag->offset; 75218b683bfSPaolo Abeni dfrag_collapsed = mptcp_frag_can_collapse_to(msk, pfrag, dfrag); 75318b683bfSPaolo Abeni if (!dfrag_collapsed) { 75418b683bfSPaolo Abeni dfrag = mptcp_carve_data_frag(msk, pfrag, offset); 75518b683bfSPaolo Abeni offset = dfrag->offset; 75618b683bfSPaolo Abeni frag_truesize = dfrag->overhead; 75718b683bfSPaolo Abeni } 75818b683bfSPaolo Abeni psize = min_t(size_t, pfrag->size - offset, avail_size); 75957040755SPaolo Abeni 76057040755SPaolo Abeni /* Copy to page */ 7616d0060f6SMat Martineau pr_debug("left=%zu", msg_data_left(msg)); 76218b683bfSPaolo Abeni psize = copy_page_from_iter(pfrag->page, offset, 7633f8e0aaeSPaolo Abeni min_t(size_t, msg_data_left(msg), 7643f8e0aaeSPaolo Abeni psize), 7656d0060f6SMat Martineau &msg->msg_iter); 7666d0060f6SMat Martineau pr_debug("left=%zu", msg_data_left(msg)); 7676d0060f6SMat Martineau if (!psize) 7686d0060f6SMat Martineau return -EINVAL; 7696d0060f6SMat Martineau 77035759383SFlorian Westphal if (!sk_wmem_schedule(sk, psize + dfrag->overhead)) { 77135759383SFlorian Westphal iov_iter_revert(&msg->msg_iter, psize); 772d027236cSPaolo Abeni return -ENOMEM; 77335759383SFlorian Westphal } 7743f8e0aaeSPaolo Abeni } else { 7753f8e0aaeSPaolo Abeni offset = dfrag->offset; 7763f8e0aaeSPaolo Abeni psize = min_t(size_t, dfrag->data_len, avail_size); 7773f8e0aaeSPaolo Abeni } 778d027236cSPaolo Abeni 77957040755SPaolo Abeni /* tell the TCP stack to delay the push so that we can safely 78057040755SPaolo Abeni * access the skb after the sendpages call 7816d0060f6SMat Martineau */ 7823f8e0aaeSPaolo Abeni ret = do_tcp_sendpages(ssk, page, offset, psize, 78372511aabSFlorian Westphal msg->msg_flags | MSG_SENDPAGE_NOTLAST | MSG_DONTWAIT); 78435759383SFlorian Westphal if (ret <= 0) { 785b3b2854dSFlorian Westphal if (!retransmission) 78635759383SFlorian Westphal iov_iter_revert(&msg->msg_iter, psize); 7876d0060f6SMat Martineau return ret; 78835759383SFlorian Westphal } 78918b683bfSPaolo Abeni 79018b683bfSPaolo Abeni frag_truesize += ret; 7913f8e0aaeSPaolo Abeni if (!retransmission) { 7926d0060f6SMat Martineau if (unlikely(ret < psize)) 7936d0060f6SMat Martineau iov_iter_revert(&msg->msg_iter, psize - ret); 7946d0060f6SMat Martineau 79518b683bfSPaolo Abeni /* send successful, keep track of sent data for mptcp-level 79618b683bfSPaolo Abeni * retransmission 79718b683bfSPaolo Abeni */ 79818b683bfSPaolo Abeni dfrag->data_len += ret; 79918b683bfSPaolo Abeni if (!dfrag_collapsed) { 80018b683bfSPaolo Abeni get_page(dfrag->page); 80118b683bfSPaolo Abeni list_add_tail(&dfrag->list, &msk->rtx_queue); 8023f8e0aaeSPaolo Abeni sk_wmem_queued_add(sk, frag_truesize); 8033f8e0aaeSPaolo Abeni } else { 8043f8e0aaeSPaolo Abeni sk_wmem_queued_add(sk, ret); 80518b683bfSPaolo Abeni } 80618b683bfSPaolo Abeni 807d027236cSPaolo Abeni /* charge data on mptcp rtx queue to the master socket 808d027236cSPaolo Abeni * Note: we charge such data both to sk and ssk 809d027236cSPaolo Abeni */ 810d027236cSPaolo Abeni sk->sk_forward_alloc -= frag_truesize; 8113f8e0aaeSPaolo Abeni } 812d027236cSPaolo Abeni 81357040755SPaolo Abeni /* if the tail skb extension is still the cached one, collapsing 81457040755SPaolo Abeni * really happened. Note: we can't check for 'same skb' as the sk_buff 81557040755SPaolo Abeni * hdr on tail can be transmitted, freed and re-allocated by the 81657040755SPaolo Abeni * do_tcp_sendpages() call 81757040755SPaolo Abeni */ 81857040755SPaolo Abeni tail = tcp_write_queue_tail(ssk); 81957040755SPaolo Abeni if (mpext && tail && mpext == skb_ext_find(tail, SKB_EXT_MPTCP)) { 82057040755SPaolo Abeni WARN_ON_ONCE(!can_collapse); 82157040755SPaolo Abeni mpext->data_len += ret; 82257040755SPaolo Abeni goto out; 82357040755SPaolo Abeni } 82457040755SPaolo Abeni 8256d0060f6SMat Martineau skb = tcp_write_queue_tail(ssk); 8266d0060f6SMat Martineau mpext = __skb_ext_set(skb, SKB_EXT_MPTCP, msk->cached_ext); 8276d0060f6SMat Martineau msk->cached_ext = NULL; 8286d0060f6SMat Martineau 8296d0060f6SMat Martineau memset(mpext, 0, sizeof(*mpext)); 8303f8e0aaeSPaolo Abeni mpext->data_seq = *write_seq; 8316d0060f6SMat Martineau mpext->subflow_seq = mptcp_subflow_ctx(ssk)->rel_write_seq; 8326d0060f6SMat Martineau mpext->data_len = ret; 8336d0060f6SMat Martineau mpext->use_map = 1; 8346d0060f6SMat Martineau mpext->dsn64 = 1; 8356d0060f6SMat Martineau 8366d0060f6SMat Martineau pr_debug("data_seq=%llu subflow_seq=%u data_len=%u dsn64=%d", 8376d0060f6SMat Martineau mpext->data_seq, mpext->subflow_seq, mpext->data_len, 8386d0060f6SMat Martineau mpext->dsn64); 8396d0060f6SMat Martineau 84057040755SPaolo Abeni out: 8413f8e0aaeSPaolo Abeni if (!retransmission) 84218b683bfSPaolo Abeni pfrag->offset += frag_truesize; 843721e9089SMat Martineau WRITE_ONCE(*write_seq, *write_seq + ret); 8446d0060f6SMat Martineau mptcp_subflow_ctx(ssk)->rel_write_seq += ret; 8456d0060f6SMat Martineau 8466d0060f6SMat Martineau return ret; 8476d0060f6SMat Martineau } 8486d0060f6SMat Martineau 84963561a40SPaolo Abeni static void mptcp_nospace(struct mptcp_sock *msk) 850a0e17064SFlorian Westphal { 85163561a40SPaolo Abeni struct mptcp_subflow_context *subflow; 85263561a40SPaolo Abeni 853a0e17064SFlorian Westphal clear_bit(MPTCP_SEND_SPACE, &msk->flags); 854a0e17064SFlorian Westphal smp_mb__after_atomic(); /* msk->flags is changed by write_space cb */ 855a0e17064SFlorian Westphal 85663561a40SPaolo Abeni mptcp_for_each_subflow(msk, subflow) { 85763561a40SPaolo Abeni struct sock *ssk = mptcp_subflow_tcp_sock(subflow); 85863561a40SPaolo Abeni struct socket *sock = READ_ONCE(ssk->sk_socket); 85963561a40SPaolo Abeni 86063561a40SPaolo Abeni /* enables ssk->write_space() callbacks */ 86163561a40SPaolo Abeni if (sock) 862a0e17064SFlorian Westphal set_bit(SOCK_NOSPACE, &sock->flags); 863a0e17064SFlorian Westphal } 86463561a40SPaolo Abeni } 865a0e17064SFlorian Westphal 866da51aef5SPaolo Abeni static struct sock *mptcp_subflow_get_send(struct mptcp_sock *msk, 867da51aef5SPaolo Abeni u32 *sndbuf) 868f296234cSPeter Krystad { 869f296234cSPeter Krystad struct mptcp_subflow_context *subflow; 87063561a40SPaolo Abeni struct sock *sk = (struct sock *)msk; 871f296234cSPeter Krystad struct sock *backup = NULL; 87263561a40SPaolo Abeni bool free; 873f296234cSPeter Krystad 87463561a40SPaolo Abeni sock_owned_by_me(sk); 875f296234cSPeter Krystad 876da51aef5SPaolo Abeni *sndbuf = 0; 877149f7c71SFlorian Westphal if (!mptcp_ext_cache_refill(msk)) 878149f7c71SFlorian Westphal return NULL; 879149f7c71SFlorian Westphal 880f296234cSPeter Krystad mptcp_for_each_subflow(msk, subflow) { 881f296234cSPeter Krystad struct sock *ssk = mptcp_subflow_tcp_sock(subflow); 882f296234cSPeter Krystad 88363561a40SPaolo Abeni free = sk_stream_is_writeable(subflow->tcp_sock); 88463561a40SPaolo Abeni if (!free) { 88563561a40SPaolo Abeni mptcp_nospace(msk); 886f296234cSPeter Krystad return NULL; 887f296234cSPeter Krystad } 888f296234cSPeter Krystad 889da51aef5SPaolo Abeni *sndbuf = max(tcp_sk(ssk)->snd_wnd, *sndbuf); 890f296234cSPeter Krystad if (subflow->backup) { 891f296234cSPeter Krystad if (!backup) 892f296234cSPeter Krystad backup = ssk; 893f296234cSPeter Krystad 894f296234cSPeter Krystad continue; 895f296234cSPeter Krystad } 896f296234cSPeter Krystad 897f296234cSPeter Krystad return ssk; 898f296234cSPeter Krystad } 899f296234cSPeter Krystad 900f296234cSPeter Krystad return backup; 901f296234cSPeter Krystad } 902f296234cSPeter Krystad 90363561a40SPaolo Abeni static void ssk_check_wmem(struct mptcp_sock *msk) 9041891c4a0SFlorian Westphal { 90563561a40SPaolo Abeni if (unlikely(!mptcp_is_writeable(msk))) 90663561a40SPaolo Abeni mptcp_nospace(msk); 9071891c4a0SFlorian Westphal } 9081891c4a0SFlorian Westphal 909f870fa0bSMat Martineau static int mptcp_sendmsg(struct sock *sk, struct msghdr *msg, size_t len) 910f870fa0bSMat Martineau { 91157040755SPaolo Abeni int mss_now = 0, size_goal = 0, ret = 0; 912f870fa0bSMat Martineau struct mptcp_sock *msk = mptcp_sk(sk); 91317091708SFlorian Westphal struct page_frag *pfrag; 9146d0060f6SMat Martineau size_t copied = 0; 915cec37a6eSPeter Krystad struct sock *ssk; 916da51aef5SPaolo Abeni u32 sndbuf; 91772511aabSFlorian Westphal bool tx_ok; 9186d0060f6SMat Martineau long timeo; 919f870fa0bSMat Martineau 920f870fa0bSMat Martineau if (msg->msg_flags & ~(MSG_MORE | MSG_DONTWAIT | MSG_NOSIGNAL)) 921f870fa0bSMat Martineau return -EOPNOTSUPP; 922f870fa0bSMat Martineau 923cec37a6eSPeter Krystad lock_sock(sk); 9241954b860SMat Martineau 9251954b860SMat Martineau timeo = sock_sndtimeo(sk, msg->msg_flags & MSG_DONTWAIT); 9261954b860SMat Martineau 9271954b860SMat Martineau if ((1 << sk->sk_state) & ~(TCPF_ESTABLISHED | TCPF_CLOSE_WAIT)) { 9281954b860SMat Martineau ret = sk_stream_wait_connect(sk, &timeo); 9291954b860SMat Martineau if (ret) 9301954b860SMat Martineau goto out; 9311954b860SMat Martineau } 9321954b860SMat Martineau 93317091708SFlorian Westphal pfrag = sk_page_frag(sk); 93472511aabSFlorian Westphal restart: 93518b683bfSPaolo Abeni mptcp_clean_una(sk); 93618b683bfSPaolo Abeni 93757baaf28SMat Martineau if (sk->sk_err || (sk->sk_shutdown & SEND_SHUTDOWN)) { 93857baaf28SMat Martineau ret = -EPIPE; 93957baaf28SMat Martineau goto out; 94057baaf28SMat Martineau } 94157baaf28SMat Martineau 942ec3edaa7SPeter Krystad __mptcp_flush_join_list(msk); 943da51aef5SPaolo Abeni ssk = mptcp_subflow_get_send(msk, &sndbuf); 94417091708SFlorian Westphal while (!sk_stream_memory_free(sk) || 94517091708SFlorian Westphal !ssk || 94617091708SFlorian Westphal !mptcp_page_frag_refill(ssk, pfrag)) { 947fb529e62SFlorian Westphal if (ssk) { 948fb529e62SFlorian Westphal /* make sure retransmit timer is 949fb529e62SFlorian Westphal * running before we wait for memory. 950fb529e62SFlorian Westphal * 951fb529e62SFlorian Westphal * The retransmit timer might be needed 952fb529e62SFlorian Westphal * to make the peer send an up-to-date 953fb529e62SFlorian Westphal * MPTCP Ack. 954fb529e62SFlorian Westphal */ 955fb529e62SFlorian Westphal mptcp_set_timeout(sk, ssk); 956fb529e62SFlorian Westphal if (!mptcp_timer_pending(sk)) 957fb529e62SFlorian Westphal mptcp_reset_timer(sk); 958fb529e62SFlorian Westphal } 959fb529e62SFlorian Westphal 96063561a40SPaolo Abeni mptcp_nospace(msk); 961f296234cSPeter Krystad ret = sk_stream_wait_memory(sk, &timeo); 962f296234cSPeter Krystad if (ret) 963f296234cSPeter Krystad goto out; 964f296234cSPeter Krystad 96518b683bfSPaolo Abeni mptcp_clean_una(sk); 96618b683bfSPaolo Abeni 967da51aef5SPaolo Abeni ssk = mptcp_subflow_get_send(msk, &sndbuf); 968f296234cSPeter Krystad if (list_empty(&msk->conn_list)) { 969f296234cSPeter Krystad ret = -ENOTCONN; 970f296234cSPeter Krystad goto out; 971f296234cSPeter Krystad } 972cec37a6eSPeter Krystad } 973cec37a6eSPeter Krystad 974da51aef5SPaolo Abeni /* do auto tuning */ 975da51aef5SPaolo Abeni if (!(sk->sk_userlocks & SOCK_SNDBUF_LOCK) && 976da51aef5SPaolo Abeni sndbuf > READ_ONCE(sk->sk_sndbuf)) 977da51aef5SPaolo Abeni WRITE_ONCE(sk->sk_sndbuf, sndbuf); 978da51aef5SPaolo Abeni 9796d0060f6SMat Martineau pr_debug("conn_list->subflow=%p", ssk); 980cec37a6eSPeter Krystad 9816d0060f6SMat Martineau lock_sock(ssk); 98272511aabSFlorian Westphal tx_ok = msg_data_left(msg); 98372511aabSFlorian Westphal while (tx_ok) { 9843f8e0aaeSPaolo Abeni ret = mptcp_sendmsg_frag(sk, ssk, msg, NULL, &timeo, &mss_now, 98557040755SPaolo Abeni &size_goal); 98672511aabSFlorian Westphal if (ret < 0) { 98772511aabSFlorian Westphal if (ret == -EAGAIN && timeo > 0) { 98872511aabSFlorian Westphal mptcp_set_timeout(sk, ssk); 98972511aabSFlorian Westphal release_sock(ssk); 99072511aabSFlorian Westphal goto restart; 99172511aabSFlorian Westphal } 9926d0060f6SMat Martineau break; 99372511aabSFlorian Westphal } 9946d0060f6SMat Martineau 9956d0060f6SMat Martineau copied += ret; 996fb529e62SFlorian Westphal 99772511aabSFlorian Westphal tx_ok = msg_data_left(msg); 99872511aabSFlorian Westphal if (!tx_ok) 99972511aabSFlorian Westphal break; 100072511aabSFlorian Westphal 1001149f7c71SFlorian Westphal if (!sk_stream_memory_free(ssk) || 100217091708SFlorian Westphal !mptcp_page_frag_refill(ssk, pfrag) || 1003149f7c71SFlorian Westphal !mptcp_ext_cache_refill(msk)) { 100472511aabSFlorian Westphal tcp_push(ssk, msg->msg_flags, mss_now, 100572511aabSFlorian Westphal tcp_sk(ssk)->nonagle, size_goal); 100672511aabSFlorian Westphal mptcp_set_timeout(sk, ssk); 100772511aabSFlorian Westphal release_sock(ssk); 100872511aabSFlorian Westphal goto restart; 100972511aabSFlorian Westphal } 101072511aabSFlorian Westphal 1011fb529e62SFlorian Westphal /* memory is charged to mptcp level socket as well, i.e. 1012fb529e62SFlorian Westphal * if msg is very large, mptcp socket may run out of buffer 1013fb529e62SFlorian Westphal * space. mptcp_clean_una() will release data that has 1014fb529e62SFlorian Westphal * been acked at mptcp level in the mean time, so there is 1015fb529e62SFlorian Westphal * a good chance we can continue sending data right away. 101672511aabSFlorian Westphal * 101772511aabSFlorian Westphal * Normally, when the tcp subflow can accept more data, then 101872511aabSFlorian Westphal * so can the MPTCP socket. However, we need to cope with 101972511aabSFlorian Westphal * peers that might lag behind in their MPTCP-level 102072511aabSFlorian Westphal * acknowledgements, i.e. data might have been acked at 102172511aabSFlorian Westphal * tcp level only. So, we must also check the MPTCP socket 102272511aabSFlorian Westphal * limits before we send more data. 1023fb529e62SFlorian Westphal */ 1024fb529e62SFlorian Westphal if (unlikely(!sk_stream_memory_free(sk))) { 1025fb529e62SFlorian Westphal tcp_push(ssk, msg->msg_flags, mss_now, 1026fb529e62SFlorian Westphal tcp_sk(ssk)->nonagle, size_goal); 1027fb529e62SFlorian Westphal mptcp_clean_una(sk); 1028fb529e62SFlorian Westphal if (!sk_stream_memory_free(sk)) { 1029fb529e62SFlorian Westphal /* can't send more for now, need to wait for 1030fb529e62SFlorian Westphal * MPTCP-level ACKs from peer. 1031fb529e62SFlorian Westphal * 1032fb529e62SFlorian Westphal * Wakeup will happen via mptcp_clean_una(). 1033fb529e62SFlorian Westphal */ 1034fb529e62SFlorian Westphal mptcp_set_timeout(sk, ssk); 1035fb529e62SFlorian Westphal release_sock(ssk); 10361cec170dSFlorian Westphal goto restart; 1037fb529e62SFlorian Westphal } 1038fb529e62SFlorian Westphal } 10396d0060f6SMat Martineau } 10406d0060f6SMat Martineau 1041b51f9b80SPaolo Abeni mptcp_set_timeout(sk, ssk); 104257040755SPaolo Abeni if (copied) { 104357040755SPaolo Abeni tcp_push(ssk, msg->msg_flags, mss_now, tcp_sk(ssk)->nonagle, 104457040755SPaolo Abeni size_goal); 1045b51f9b80SPaolo Abeni 1046b51f9b80SPaolo Abeni /* start the timer, if it's not pending */ 1047b51f9b80SPaolo Abeni if (!mptcp_timer_pending(sk)) 1048b51f9b80SPaolo Abeni mptcp_reset_timer(sk); 104957040755SPaolo Abeni } 10506d0060f6SMat Martineau 10516d0060f6SMat Martineau release_sock(ssk); 10521954b860SMat Martineau out: 105363561a40SPaolo Abeni ssk_check_wmem(msk); 1054cec37a6eSPeter Krystad release_sock(sk); 10558555c6bfSPaolo Abeni return copied ? : ret; 1056f870fa0bSMat Martineau } 1057f870fa0bSMat Martineau 10587a6a6cbcSPaolo Abeni static void mptcp_wait_data(struct sock *sk, long *timeo) 10597a6a6cbcSPaolo Abeni { 10607a6a6cbcSPaolo Abeni DEFINE_WAIT_FUNC(wait, woken_wake_function); 10617a6a6cbcSPaolo Abeni struct mptcp_sock *msk = mptcp_sk(sk); 10627a6a6cbcSPaolo Abeni 10637a6a6cbcSPaolo Abeni add_wait_queue(sk_sleep(sk), &wait); 10647a6a6cbcSPaolo Abeni sk_set_bit(SOCKWQ_ASYNC_WAITDATA, sk); 10657a6a6cbcSPaolo Abeni 10667a6a6cbcSPaolo Abeni sk_wait_event(sk, timeo, 10677a6a6cbcSPaolo Abeni test_and_clear_bit(MPTCP_DATA_READY, &msk->flags), &wait); 10687a6a6cbcSPaolo Abeni 10697a6a6cbcSPaolo Abeni sk_clear_bit(SOCKWQ_ASYNC_WAITDATA, sk); 10707a6a6cbcSPaolo Abeni remove_wait_queue(sk_sleep(sk), &wait); 10717a6a6cbcSPaolo Abeni } 10727a6a6cbcSPaolo Abeni 10736771bfd9SFlorian Westphal static int __mptcp_recvmsg_mskq(struct mptcp_sock *msk, 10746771bfd9SFlorian Westphal struct msghdr *msg, 10756771bfd9SFlorian Westphal size_t len) 10766771bfd9SFlorian Westphal { 10776771bfd9SFlorian Westphal struct sock *sk = (struct sock *)msk; 10786771bfd9SFlorian Westphal struct sk_buff *skb; 10796771bfd9SFlorian Westphal int copied = 0; 10806771bfd9SFlorian Westphal 10816771bfd9SFlorian Westphal while ((skb = skb_peek(&sk->sk_receive_queue)) != NULL) { 10826771bfd9SFlorian Westphal u32 offset = MPTCP_SKB_CB(skb)->offset; 10836771bfd9SFlorian Westphal u32 data_len = skb->len - offset; 10846771bfd9SFlorian Westphal u32 count = min_t(size_t, len - copied, data_len); 10856771bfd9SFlorian Westphal int err; 10866771bfd9SFlorian Westphal 10876771bfd9SFlorian Westphal err = skb_copy_datagram_msg(skb, offset, msg, count); 10886771bfd9SFlorian Westphal if (unlikely(err < 0)) { 10896771bfd9SFlorian Westphal if (!copied) 10906771bfd9SFlorian Westphal return err; 10916771bfd9SFlorian Westphal break; 10926771bfd9SFlorian Westphal } 10936771bfd9SFlorian Westphal 10946771bfd9SFlorian Westphal copied += count; 10956771bfd9SFlorian Westphal 10966771bfd9SFlorian Westphal if (count < data_len) { 10976771bfd9SFlorian Westphal MPTCP_SKB_CB(skb)->offset += count; 10986771bfd9SFlorian Westphal break; 10996771bfd9SFlorian Westphal } 11006771bfd9SFlorian Westphal 11016771bfd9SFlorian Westphal __skb_unlink(skb, &sk->sk_receive_queue); 11026771bfd9SFlorian Westphal __kfree_skb(skb); 11036771bfd9SFlorian Westphal 11046771bfd9SFlorian Westphal if (copied >= len) 11056771bfd9SFlorian Westphal break; 11066771bfd9SFlorian Westphal } 11076771bfd9SFlorian Westphal 11086771bfd9SFlorian Westphal return copied; 11096771bfd9SFlorian Westphal } 11106771bfd9SFlorian Westphal 1111a6b118feSFlorian Westphal /* receive buffer autotuning. See tcp_rcv_space_adjust for more information. 1112a6b118feSFlorian Westphal * 1113a6b118feSFlorian Westphal * Only difference: Use highest rtt estimate of the subflows in use. 1114a6b118feSFlorian Westphal */ 1115a6b118feSFlorian Westphal static void mptcp_rcv_space_adjust(struct mptcp_sock *msk, int copied) 1116a6b118feSFlorian Westphal { 1117a6b118feSFlorian Westphal struct mptcp_subflow_context *subflow; 1118a6b118feSFlorian Westphal struct sock *sk = (struct sock *)msk; 1119a6b118feSFlorian Westphal u32 time, advmss = 1; 1120a6b118feSFlorian Westphal u64 rtt_us, mstamp; 1121a6b118feSFlorian Westphal 1122a6b118feSFlorian Westphal sock_owned_by_me(sk); 1123a6b118feSFlorian Westphal 1124a6b118feSFlorian Westphal if (copied <= 0) 1125a6b118feSFlorian Westphal return; 1126a6b118feSFlorian Westphal 1127a6b118feSFlorian Westphal msk->rcvq_space.copied += copied; 1128a6b118feSFlorian Westphal 1129a6b118feSFlorian Westphal mstamp = div_u64(tcp_clock_ns(), NSEC_PER_USEC); 1130a6b118feSFlorian Westphal time = tcp_stamp_us_delta(mstamp, msk->rcvq_space.time); 1131a6b118feSFlorian Westphal 1132a6b118feSFlorian Westphal rtt_us = msk->rcvq_space.rtt_us; 1133a6b118feSFlorian Westphal if (rtt_us && time < (rtt_us >> 3)) 1134a6b118feSFlorian Westphal return; 1135a6b118feSFlorian Westphal 1136a6b118feSFlorian Westphal rtt_us = 0; 1137a6b118feSFlorian Westphal mptcp_for_each_subflow(msk, subflow) { 1138a6b118feSFlorian Westphal const struct tcp_sock *tp; 1139a6b118feSFlorian Westphal u64 sf_rtt_us; 1140a6b118feSFlorian Westphal u32 sf_advmss; 1141a6b118feSFlorian Westphal 1142a6b118feSFlorian Westphal tp = tcp_sk(mptcp_subflow_tcp_sock(subflow)); 1143a6b118feSFlorian Westphal 1144a6b118feSFlorian Westphal sf_rtt_us = READ_ONCE(tp->rcv_rtt_est.rtt_us); 1145a6b118feSFlorian Westphal sf_advmss = READ_ONCE(tp->advmss); 1146a6b118feSFlorian Westphal 1147a6b118feSFlorian Westphal rtt_us = max(sf_rtt_us, rtt_us); 1148a6b118feSFlorian Westphal advmss = max(sf_advmss, advmss); 1149a6b118feSFlorian Westphal } 1150a6b118feSFlorian Westphal 1151a6b118feSFlorian Westphal msk->rcvq_space.rtt_us = rtt_us; 1152a6b118feSFlorian Westphal if (time < (rtt_us >> 3) || rtt_us == 0) 1153a6b118feSFlorian Westphal return; 1154a6b118feSFlorian Westphal 1155a6b118feSFlorian Westphal if (msk->rcvq_space.copied <= msk->rcvq_space.space) 1156a6b118feSFlorian Westphal goto new_measure; 1157a6b118feSFlorian Westphal 1158a6b118feSFlorian Westphal if (sock_net(sk)->ipv4.sysctl_tcp_moderate_rcvbuf && 1159a6b118feSFlorian Westphal !(sk->sk_userlocks & SOCK_RCVBUF_LOCK)) { 1160a6b118feSFlorian Westphal int rcvmem, rcvbuf; 1161a6b118feSFlorian Westphal u64 rcvwin, grow; 1162a6b118feSFlorian Westphal 1163a6b118feSFlorian Westphal rcvwin = ((u64)msk->rcvq_space.copied << 1) + 16 * advmss; 1164a6b118feSFlorian Westphal 1165a6b118feSFlorian Westphal grow = rcvwin * (msk->rcvq_space.copied - msk->rcvq_space.space); 1166a6b118feSFlorian Westphal 1167a6b118feSFlorian Westphal do_div(grow, msk->rcvq_space.space); 1168a6b118feSFlorian Westphal rcvwin += (grow << 1); 1169a6b118feSFlorian Westphal 1170a6b118feSFlorian Westphal rcvmem = SKB_TRUESIZE(advmss + MAX_TCP_HEADER); 1171a6b118feSFlorian Westphal while (tcp_win_from_space(sk, rcvmem) < advmss) 1172a6b118feSFlorian Westphal rcvmem += 128; 1173a6b118feSFlorian Westphal 1174a6b118feSFlorian Westphal do_div(rcvwin, advmss); 1175a6b118feSFlorian Westphal rcvbuf = min_t(u64, rcvwin * rcvmem, 1176a6b118feSFlorian Westphal sock_net(sk)->ipv4.sysctl_tcp_rmem[2]); 1177a6b118feSFlorian Westphal 1178a6b118feSFlorian Westphal if (rcvbuf > sk->sk_rcvbuf) { 1179a6b118feSFlorian Westphal u32 window_clamp; 1180a6b118feSFlorian Westphal 1181a6b118feSFlorian Westphal window_clamp = tcp_win_from_space(sk, rcvbuf); 1182a6b118feSFlorian Westphal WRITE_ONCE(sk->sk_rcvbuf, rcvbuf); 1183a6b118feSFlorian Westphal 1184a6b118feSFlorian Westphal /* Make subflows follow along. If we do not do this, we 1185a6b118feSFlorian Westphal * get drops at subflow level if skbs can't be moved to 1186a6b118feSFlorian Westphal * the mptcp rx queue fast enough (announced rcv_win can 1187a6b118feSFlorian Westphal * exceed ssk->sk_rcvbuf). 1188a6b118feSFlorian Westphal */ 1189a6b118feSFlorian Westphal mptcp_for_each_subflow(msk, subflow) { 1190a6b118feSFlorian Westphal struct sock *ssk; 1191a6b118feSFlorian Westphal 1192a6b118feSFlorian Westphal ssk = mptcp_subflow_tcp_sock(subflow); 1193a6b118feSFlorian Westphal WRITE_ONCE(ssk->sk_rcvbuf, rcvbuf); 1194a6b118feSFlorian Westphal tcp_sk(ssk)->window_clamp = window_clamp; 1195a6b118feSFlorian Westphal } 1196a6b118feSFlorian Westphal } 1197a6b118feSFlorian Westphal } 1198a6b118feSFlorian Westphal 1199a6b118feSFlorian Westphal msk->rcvq_space.space = msk->rcvq_space.copied; 1200a6b118feSFlorian Westphal new_measure: 1201a6b118feSFlorian Westphal msk->rcvq_space.copied = 0; 1202a6b118feSFlorian Westphal msk->rcvq_space.time = mstamp; 1203a6b118feSFlorian Westphal } 1204a6b118feSFlorian Westphal 12056771bfd9SFlorian Westphal static bool __mptcp_move_skbs(struct mptcp_sock *msk) 12066771bfd9SFlorian Westphal { 12076771bfd9SFlorian Westphal unsigned int moved = 0; 12086771bfd9SFlorian Westphal bool done; 12096771bfd9SFlorian Westphal 12106771bfd9SFlorian Westphal do { 12116771bfd9SFlorian Westphal struct sock *ssk = mptcp_subflow_recv_lookup(msk); 12126771bfd9SFlorian Westphal 12136771bfd9SFlorian Westphal if (!ssk) 12146771bfd9SFlorian Westphal break; 12156771bfd9SFlorian Westphal 12166771bfd9SFlorian Westphal lock_sock(ssk); 12176771bfd9SFlorian Westphal done = __mptcp_move_skbs_from_subflow(msk, ssk, &moved); 12186771bfd9SFlorian Westphal release_sock(ssk); 12196771bfd9SFlorian Westphal } while (!done); 12206771bfd9SFlorian Westphal 12216771bfd9SFlorian Westphal return moved > 0; 12226771bfd9SFlorian Westphal } 12236771bfd9SFlorian Westphal 1224f870fa0bSMat Martineau static int mptcp_recvmsg(struct sock *sk, struct msghdr *msg, size_t len, 1225f870fa0bSMat Martineau int nonblock, int flags, int *addr_len) 1226f870fa0bSMat Martineau { 1227f870fa0bSMat Martineau struct mptcp_sock *msk = mptcp_sk(sk); 1228cec37a6eSPeter Krystad int copied = 0; 12297a6a6cbcSPaolo Abeni int target; 12307a6a6cbcSPaolo Abeni long timeo; 1231f870fa0bSMat Martineau 1232f870fa0bSMat Martineau if (msg->msg_flags & ~(MSG_WAITALL | MSG_DONTWAIT)) 1233f870fa0bSMat Martineau return -EOPNOTSUPP; 1234f870fa0bSMat Martineau 1235cec37a6eSPeter Krystad lock_sock(sk); 12367a6a6cbcSPaolo Abeni timeo = sock_rcvtimeo(sk, nonblock); 12377a6a6cbcSPaolo Abeni 12387a6a6cbcSPaolo Abeni len = min_t(size_t, len, INT_MAX); 12397a6a6cbcSPaolo Abeni target = sock_rcvlowat(sk, flags & MSG_WAITALL, len); 1240ec3edaa7SPeter Krystad __mptcp_flush_join_list(msk); 12417a6a6cbcSPaolo Abeni 12426771bfd9SFlorian Westphal while (len > (size_t)copied) { 12437a6a6cbcSPaolo Abeni int bytes_read; 12447a6a6cbcSPaolo Abeni 12456771bfd9SFlorian Westphal bytes_read = __mptcp_recvmsg_mskq(msk, msg, len - copied); 12466771bfd9SFlorian Westphal if (unlikely(bytes_read < 0)) { 12477a6a6cbcSPaolo Abeni if (!copied) 12487a6a6cbcSPaolo Abeni copied = bytes_read; 12496771bfd9SFlorian Westphal goto out_err; 1250cec37a6eSPeter Krystad } 1251cec37a6eSPeter Krystad 12527a6a6cbcSPaolo Abeni copied += bytes_read; 12537a6a6cbcSPaolo Abeni 12546771bfd9SFlorian Westphal if (skb_queue_empty(&sk->sk_receive_queue) && 12556771bfd9SFlorian Westphal __mptcp_move_skbs(msk)) 12566771bfd9SFlorian Westphal continue; 12577a6a6cbcSPaolo Abeni 12587a6a6cbcSPaolo Abeni /* only the master socket status is relevant here. The exit 12597a6a6cbcSPaolo Abeni * conditions mirror closely tcp_recvmsg() 12607a6a6cbcSPaolo Abeni */ 12617a6a6cbcSPaolo Abeni if (copied >= target) 12627a6a6cbcSPaolo Abeni break; 12637a6a6cbcSPaolo Abeni 12647a6a6cbcSPaolo Abeni if (copied) { 12657a6a6cbcSPaolo Abeni if (sk->sk_err || 12667a6a6cbcSPaolo Abeni sk->sk_state == TCP_CLOSE || 12677a6a6cbcSPaolo Abeni (sk->sk_shutdown & RCV_SHUTDOWN) || 12687a6a6cbcSPaolo Abeni !timeo || 12697a6a6cbcSPaolo Abeni signal_pending(current)) 12707a6a6cbcSPaolo Abeni break; 12717a6a6cbcSPaolo Abeni } else { 12727a6a6cbcSPaolo Abeni if (sk->sk_err) { 12737a6a6cbcSPaolo Abeni copied = sock_error(sk); 12747a6a6cbcSPaolo Abeni break; 12757a6a6cbcSPaolo Abeni } 12767a6a6cbcSPaolo Abeni 12775969856aSPaolo Abeni if (test_and_clear_bit(MPTCP_WORK_EOF, &msk->flags)) 12785969856aSPaolo Abeni mptcp_check_for_eof(msk); 12795969856aSPaolo Abeni 12807a6a6cbcSPaolo Abeni if (sk->sk_shutdown & RCV_SHUTDOWN) 12817a6a6cbcSPaolo Abeni break; 12827a6a6cbcSPaolo Abeni 12837a6a6cbcSPaolo Abeni if (sk->sk_state == TCP_CLOSE) { 12847a6a6cbcSPaolo Abeni copied = -ENOTCONN; 12857a6a6cbcSPaolo Abeni break; 12867a6a6cbcSPaolo Abeni } 12877a6a6cbcSPaolo Abeni 12887a6a6cbcSPaolo Abeni if (!timeo) { 12897a6a6cbcSPaolo Abeni copied = -EAGAIN; 12907a6a6cbcSPaolo Abeni break; 12917a6a6cbcSPaolo Abeni } 12927a6a6cbcSPaolo Abeni 12937a6a6cbcSPaolo Abeni if (signal_pending(current)) { 12947a6a6cbcSPaolo Abeni copied = sock_intr_errno(timeo); 12957a6a6cbcSPaolo Abeni break; 12967a6a6cbcSPaolo Abeni } 12977a6a6cbcSPaolo Abeni } 12987a6a6cbcSPaolo Abeni 12997a6a6cbcSPaolo Abeni pr_debug("block timeout %ld", timeo); 13007a6a6cbcSPaolo Abeni mptcp_wait_data(sk, &timeo); 13017a6a6cbcSPaolo Abeni } 13027a6a6cbcSPaolo Abeni 13036771bfd9SFlorian Westphal if (skb_queue_empty(&sk->sk_receive_queue)) { 13046771bfd9SFlorian Westphal /* entire backlog drained, clear DATA_READY. */ 13057a6a6cbcSPaolo Abeni clear_bit(MPTCP_DATA_READY, &msk->flags); 13067a6a6cbcSPaolo Abeni 13076771bfd9SFlorian Westphal /* .. race-breaker: ssk might have gotten new data 13086771bfd9SFlorian Westphal * after last __mptcp_move_skbs() returned false. 13097a6a6cbcSPaolo Abeni */ 13106771bfd9SFlorian Westphal if (unlikely(__mptcp_move_skbs(msk))) 13116771bfd9SFlorian Westphal set_bit(MPTCP_DATA_READY, &msk->flags); 13126771bfd9SFlorian Westphal } else if (unlikely(!test_bit(MPTCP_DATA_READY, &msk->flags))) { 13136771bfd9SFlorian Westphal /* data to read but mptcp_wait_data() cleared DATA_READY */ 13147a6a6cbcSPaolo Abeni set_bit(MPTCP_DATA_READY, &msk->flags); 13157a6a6cbcSPaolo Abeni } 13166771bfd9SFlorian Westphal out_err: 13176719331cSPaolo Abeni pr_debug("msk=%p data_ready=%d rx queue empty=%d copied=%d", 13186719331cSPaolo Abeni msk, test_bit(MPTCP_DATA_READY, &msk->flags), 13196719331cSPaolo Abeni skb_queue_empty(&sk->sk_receive_queue), copied); 1320a6b118feSFlorian Westphal mptcp_rcv_space_adjust(msk, copied); 1321a6b118feSFlorian Westphal 1322cec37a6eSPeter Krystad release_sock(sk); 1323cec37a6eSPeter Krystad return copied; 1324cec37a6eSPeter Krystad } 1325cec37a6eSPeter Krystad 1326b51f9b80SPaolo Abeni static void mptcp_retransmit_handler(struct sock *sk) 1327b51f9b80SPaolo Abeni { 1328b51f9b80SPaolo Abeni struct mptcp_sock *msk = mptcp_sk(sk); 1329b51f9b80SPaolo Abeni 1330c7529392SMat Martineau if (atomic64_read(&msk->snd_una) == READ_ONCE(msk->write_seq)) { 1331b51f9b80SPaolo Abeni mptcp_stop_timer(sk); 13323b1d6210SPaolo Abeni } else { 13333b1d6210SPaolo Abeni set_bit(MPTCP_WORK_RTX, &msk->flags); 13343b1d6210SPaolo Abeni if (schedule_work(&msk->work)) 13353b1d6210SPaolo Abeni sock_hold(sk); 13363b1d6210SPaolo Abeni } 1337b51f9b80SPaolo Abeni } 1338b51f9b80SPaolo Abeni 1339b51f9b80SPaolo Abeni static void mptcp_retransmit_timer(struct timer_list *t) 1340b51f9b80SPaolo Abeni { 1341b51f9b80SPaolo Abeni struct inet_connection_sock *icsk = from_timer(icsk, t, 1342b51f9b80SPaolo Abeni icsk_retransmit_timer); 1343b51f9b80SPaolo Abeni struct sock *sk = &icsk->icsk_inet.sk; 1344b51f9b80SPaolo Abeni 1345b51f9b80SPaolo Abeni bh_lock_sock(sk); 1346b51f9b80SPaolo Abeni if (!sock_owned_by_user(sk)) { 1347b51f9b80SPaolo Abeni mptcp_retransmit_handler(sk); 1348b51f9b80SPaolo Abeni } else { 1349b51f9b80SPaolo Abeni /* delegate our work to tcp_release_cb() */ 1350b51f9b80SPaolo Abeni if (!test_and_set_bit(TCP_WRITE_TIMER_DEFERRED, 1351b51f9b80SPaolo Abeni &sk->sk_tsq_flags)) 1352b51f9b80SPaolo Abeni sock_hold(sk); 1353b51f9b80SPaolo Abeni } 1354b51f9b80SPaolo Abeni bh_unlock_sock(sk); 1355b51f9b80SPaolo Abeni sock_put(sk); 1356b51f9b80SPaolo Abeni } 1357b51f9b80SPaolo Abeni 13583b1d6210SPaolo Abeni /* Find an idle subflow. Return NULL if there is unacked data at tcp 13593b1d6210SPaolo Abeni * level. 13603b1d6210SPaolo Abeni * 13613b1d6210SPaolo Abeni * A backup subflow is returned only if that is the only kind available. 13623b1d6210SPaolo Abeni */ 13633b1d6210SPaolo Abeni static struct sock *mptcp_subflow_get_retrans(const struct mptcp_sock *msk) 13643b1d6210SPaolo Abeni { 13653b1d6210SPaolo Abeni struct mptcp_subflow_context *subflow; 13663b1d6210SPaolo Abeni struct sock *backup = NULL; 13673b1d6210SPaolo Abeni 13683b1d6210SPaolo Abeni sock_owned_by_me((const struct sock *)msk); 13693b1d6210SPaolo Abeni 13703b1d6210SPaolo Abeni mptcp_for_each_subflow(msk, subflow) { 13713b1d6210SPaolo Abeni struct sock *ssk = mptcp_subflow_tcp_sock(subflow); 13723b1d6210SPaolo Abeni 13733b1d6210SPaolo Abeni /* still data outstanding at TCP level? Don't retransmit. */ 13743b1d6210SPaolo Abeni if (!tcp_write_queue_empty(ssk)) 13753b1d6210SPaolo Abeni return NULL; 13763b1d6210SPaolo Abeni 13773b1d6210SPaolo Abeni if (subflow->backup) { 13783b1d6210SPaolo Abeni if (!backup) 13793b1d6210SPaolo Abeni backup = ssk; 13803b1d6210SPaolo Abeni continue; 13813b1d6210SPaolo Abeni } 13823b1d6210SPaolo Abeni 13833b1d6210SPaolo Abeni return ssk; 13843b1d6210SPaolo Abeni } 13853b1d6210SPaolo Abeni 13863b1d6210SPaolo Abeni return backup; 13873b1d6210SPaolo Abeni } 13883b1d6210SPaolo Abeni 1389cec37a6eSPeter Krystad /* subflow sockets can be either outgoing (connect) or incoming 1390cec37a6eSPeter Krystad * (accept). 1391cec37a6eSPeter Krystad * 1392cec37a6eSPeter Krystad * Outgoing subflows use in-kernel sockets. 1393cec37a6eSPeter Krystad * Incoming subflows do not have their own 'struct socket' allocated, 1394cec37a6eSPeter Krystad * so we need to use tcp_close() after detaching them from the mptcp 1395cec37a6eSPeter Krystad * parent socket. 1396cec37a6eSPeter Krystad */ 1397cec37a6eSPeter Krystad static void __mptcp_close_ssk(struct sock *sk, struct sock *ssk, 1398cec37a6eSPeter Krystad struct mptcp_subflow_context *subflow, 1399cec37a6eSPeter Krystad long timeout) 1400cec37a6eSPeter Krystad { 1401cec37a6eSPeter Krystad struct socket *sock = READ_ONCE(ssk->sk_socket); 1402cec37a6eSPeter Krystad 1403cec37a6eSPeter Krystad list_del(&subflow->node); 1404cec37a6eSPeter Krystad 1405cec37a6eSPeter Krystad if (sock && sock != sk->sk_socket) { 1406cec37a6eSPeter Krystad /* outgoing subflow */ 1407cec37a6eSPeter Krystad sock_release(sock); 1408cec37a6eSPeter Krystad } else { 1409cec37a6eSPeter Krystad /* incoming subflow */ 1410cec37a6eSPeter Krystad tcp_close(ssk, timeout); 1411cec37a6eSPeter Krystad } 1412f870fa0bSMat Martineau } 1413f870fa0bSMat Martineau 1414dc24f8b4SPaolo Abeni static unsigned int mptcp_sync_mss(struct sock *sk, u32 pmtu) 1415dc24f8b4SPaolo Abeni { 1416dc24f8b4SPaolo Abeni return 0; 1417dc24f8b4SPaolo Abeni } 1418dc24f8b4SPaolo Abeni 1419b416268bSFlorian Westphal static void pm_work(struct mptcp_sock *msk) 1420b416268bSFlorian Westphal { 1421b416268bSFlorian Westphal struct mptcp_pm_data *pm = &msk->pm; 1422b416268bSFlorian Westphal 1423b416268bSFlorian Westphal spin_lock_bh(&msk->pm.lock); 1424b416268bSFlorian Westphal 1425b416268bSFlorian Westphal pr_debug("msk=%p status=%x", msk, pm->status); 1426b416268bSFlorian Westphal if (pm->status & BIT(MPTCP_PM_ADD_ADDR_RECEIVED)) { 1427b416268bSFlorian Westphal pm->status &= ~BIT(MPTCP_PM_ADD_ADDR_RECEIVED); 1428b416268bSFlorian Westphal mptcp_pm_nl_add_addr_received(msk); 1429b416268bSFlorian Westphal } 1430b416268bSFlorian Westphal if (pm->status & BIT(MPTCP_PM_ESTABLISHED)) { 1431b416268bSFlorian Westphal pm->status &= ~BIT(MPTCP_PM_ESTABLISHED); 1432b416268bSFlorian Westphal mptcp_pm_nl_fully_established(msk); 1433b416268bSFlorian Westphal } 1434b416268bSFlorian Westphal if (pm->status & BIT(MPTCP_PM_SUBFLOW_ESTABLISHED)) { 1435b416268bSFlorian Westphal pm->status &= ~BIT(MPTCP_PM_SUBFLOW_ESTABLISHED); 1436b416268bSFlorian Westphal mptcp_pm_nl_subflow_established(msk); 1437b416268bSFlorian Westphal } 1438b416268bSFlorian Westphal 1439b416268bSFlorian Westphal spin_unlock_bh(&msk->pm.lock); 1440b416268bSFlorian Westphal } 1441b416268bSFlorian Westphal 144280992017SPaolo Abeni static void mptcp_worker(struct work_struct *work) 144380992017SPaolo Abeni { 144480992017SPaolo Abeni struct mptcp_sock *msk = container_of(work, struct mptcp_sock, work); 14453b1d6210SPaolo Abeni struct sock *ssk, *sk = &msk->sk.icsk_inet.sk; 1446149f7c71SFlorian Westphal int orig_len, orig_offset, mss_now = 0, size_goal = 0; 14473b1d6210SPaolo Abeni struct mptcp_data_frag *dfrag; 14483b1d6210SPaolo Abeni u64 orig_write_seq; 14493b1d6210SPaolo Abeni size_t copied = 0; 1450b3b2854dSFlorian Westphal struct msghdr msg = { 1451b3b2854dSFlorian Westphal .msg_flags = MSG_DONTWAIT, 1452b3b2854dSFlorian Westphal }; 14533b1d6210SPaolo Abeni long timeo = 0; 145480992017SPaolo Abeni 145580992017SPaolo Abeni lock_sock(sk); 14563b1d6210SPaolo Abeni mptcp_clean_una(sk); 145743b54c6eSMat Martineau mptcp_check_data_fin_ack(sk); 1458ec3edaa7SPeter Krystad __mptcp_flush_join_list(msk); 14596771bfd9SFlorian Westphal __mptcp_move_skbs(msk); 14603b1d6210SPaolo Abeni 1461b416268bSFlorian Westphal if (msk->pm.status) 1462b416268bSFlorian Westphal pm_work(msk); 1463b416268bSFlorian Westphal 146459832e24SFlorian Westphal if (test_and_clear_bit(MPTCP_WORK_EOF, &msk->flags)) 146559832e24SFlorian Westphal mptcp_check_for_eof(msk); 146659832e24SFlorian Westphal 146743b54c6eSMat Martineau mptcp_check_data_fin(sk); 146843b54c6eSMat Martineau 14693b1d6210SPaolo Abeni if (!test_and_clear_bit(MPTCP_WORK_RTX, &msk->flags)) 14703b1d6210SPaolo Abeni goto unlock; 14713b1d6210SPaolo Abeni 14723b1d6210SPaolo Abeni dfrag = mptcp_rtx_head(sk); 14733b1d6210SPaolo Abeni if (!dfrag) 14743b1d6210SPaolo Abeni goto unlock; 14753b1d6210SPaolo Abeni 1476149f7c71SFlorian Westphal if (!mptcp_ext_cache_refill(msk)) 1477149f7c71SFlorian Westphal goto reset_unlock; 1478149f7c71SFlorian Westphal 14793b1d6210SPaolo Abeni ssk = mptcp_subflow_get_retrans(msk); 14803b1d6210SPaolo Abeni if (!ssk) 14813b1d6210SPaolo Abeni goto reset_unlock; 14823b1d6210SPaolo Abeni 14833b1d6210SPaolo Abeni lock_sock(ssk); 14843b1d6210SPaolo Abeni 14853b1d6210SPaolo Abeni orig_len = dfrag->data_len; 14863b1d6210SPaolo Abeni orig_offset = dfrag->offset; 14873b1d6210SPaolo Abeni orig_write_seq = dfrag->data_seq; 14883b1d6210SPaolo Abeni while (dfrag->data_len > 0) { 1489149f7c71SFlorian Westphal int ret = mptcp_sendmsg_frag(sk, ssk, &msg, dfrag, &timeo, 1490149f7c71SFlorian Westphal &mss_now, &size_goal); 14913b1d6210SPaolo Abeni if (ret < 0) 14923b1d6210SPaolo Abeni break; 14933b1d6210SPaolo Abeni 1494fc518953SFlorian Westphal MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_RETRANSSEGS); 14953b1d6210SPaolo Abeni copied += ret; 14963b1d6210SPaolo Abeni dfrag->data_len -= ret; 14973b1d6210SPaolo Abeni dfrag->offset += ret; 1498149f7c71SFlorian Westphal 1499149f7c71SFlorian Westphal if (!mptcp_ext_cache_refill(msk)) 1500149f7c71SFlorian Westphal break; 15013b1d6210SPaolo Abeni } 15023b1d6210SPaolo Abeni if (copied) 15033b1d6210SPaolo Abeni tcp_push(ssk, msg.msg_flags, mss_now, tcp_sk(ssk)->nonagle, 15043b1d6210SPaolo Abeni size_goal); 15053b1d6210SPaolo Abeni 15063b1d6210SPaolo Abeni dfrag->data_seq = orig_write_seq; 15073b1d6210SPaolo Abeni dfrag->offset = orig_offset; 15083b1d6210SPaolo Abeni dfrag->data_len = orig_len; 15093b1d6210SPaolo Abeni 15103b1d6210SPaolo Abeni mptcp_set_timeout(sk, ssk); 15113b1d6210SPaolo Abeni release_sock(ssk); 15123b1d6210SPaolo Abeni 15133b1d6210SPaolo Abeni reset_unlock: 15143b1d6210SPaolo Abeni if (!mptcp_timer_pending(sk)) 15153b1d6210SPaolo Abeni mptcp_reset_timer(sk); 15163b1d6210SPaolo Abeni 15173b1d6210SPaolo Abeni unlock: 151880992017SPaolo Abeni release_sock(sk); 151980992017SPaolo Abeni sock_put(sk); 152080992017SPaolo Abeni } 152180992017SPaolo Abeni 1522784325e9SMatthieu Baerts static int __mptcp_init_sock(struct sock *sk) 1523f870fa0bSMat Martineau { 1524cec37a6eSPeter Krystad struct mptcp_sock *msk = mptcp_sk(sk); 1525cec37a6eSPeter Krystad 1526ec3edaa7SPeter Krystad spin_lock_init(&msk->join_list_lock); 1527ec3edaa7SPeter Krystad 1528cec37a6eSPeter Krystad INIT_LIST_HEAD(&msk->conn_list); 1529ec3edaa7SPeter Krystad INIT_LIST_HEAD(&msk->join_list); 153018b683bfSPaolo Abeni INIT_LIST_HEAD(&msk->rtx_queue); 15311891c4a0SFlorian Westphal __set_bit(MPTCP_SEND_SPACE, &msk->flags); 153280992017SPaolo Abeni INIT_WORK(&msk->work, mptcp_worker); 1533cec37a6eSPeter Krystad 15348ab183deSPaolo Abeni msk->first = NULL; 1535dc24f8b4SPaolo Abeni inet_csk(sk)->icsk_sync_mss = mptcp_sync_mss; 15368ab183deSPaolo Abeni 15371b1c7a0eSPeter Krystad mptcp_pm_data_init(msk); 15381b1c7a0eSPeter Krystad 1539b51f9b80SPaolo Abeni /* re-use the csk retrans timer for MPTCP-level retrans */ 1540b51f9b80SPaolo Abeni timer_setup(&msk->sk.icsk_retransmit_timer, mptcp_retransmit_timer, 0); 1541b51f9b80SPaolo Abeni 1542f870fa0bSMat Martineau return 0; 1543f870fa0bSMat Martineau } 1544f870fa0bSMat Martineau 1545784325e9SMatthieu Baerts static int mptcp_init_sock(struct sock *sk) 1546784325e9SMatthieu Baerts { 1547fc518953SFlorian Westphal struct net *net = sock_net(sk); 1548fc518953SFlorian Westphal int ret; 154918b683bfSPaolo Abeni 1550fc518953SFlorian Westphal if (!mptcp_is_enabled(net)) 1551fc518953SFlorian Westphal return -ENOPROTOOPT; 1552fc518953SFlorian Westphal 1553fc518953SFlorian Westphal if (unlikely(!net->mib.mptcp_statistics) && !mptcp_mib_alloc(net)) 1554fc518953SFlorian Westphal return -ENOMEM; 1555fc518953SFlorian Westphal 1556fc518953SFlorian Westphal ret = __mptcp_init_sock(sk); 155718b683bfSPaolo Abeni if (ret) 155818b683bfSPaolo Abeni return ret; 155918b683bfSPaolo Abeni 1560fa68018dSPaolo Abeni ret = __mptcp_socket_create(mptcp_sk(sk)); 1561fa68018dSPaolo Abeni if (ret) 1562fa68018dSPaolo Abeni return ret; 1563fa68018dSPaolo Abeni 1564d027236cSPaolo Abeni sk_sockets_allocated_inc(sk); 1565a6b118feSFlorian Westphal sk->sk_rcvbuf = sock_net(sk)->ipv4.sysctl_tcp_rmem[1]; 1566da51aef5SPaolo Abeni sk->sk_sndbuf = sock_net(sk)->ipv4.sysctl_tcp_wmem[1]; 1567d027236cSPaolo Abeni 156818b683bfSPaolo Abeni return 0; 156918b683bfSPaolo Abeni } 157018b683bfSPaolo Abeni 157118b683bfSPaolo Abeni static void __mptcp_clear_xmit(struct sock *sk) 157218b683bfSPaolo Abeni { 157318b683bfSPaolo Abeni struct mptcp_sock *msk = mptcp_sk(sk); 157418b683bfSPaolo Abeni struct mptcp_data_frag *dtmp, *dfrag; 157518b683bfSPaolo Abeni 1576b51f9b80SPaolo Abeni sk_stop_timer(sk, &msk->sk.icsk_retransmit_timer); 1577b51f9b80SPaolo Abeni 157818b683bfSPaolo Abeni list_for_each_entry_safe(dfrag, dtmp, &msk->rtx_queue, list) 1579d027236cSPaolo Abeni dfrag_clear(sk, dfrag); 1580784325e9SMatthieu Baerts } 1581784325e9SMatthieu Baerts 158280992017SPaolo Abeni static void mptcp_cancel_work(struct sock *sk) 158380992017SPaolo Abeni { 158480992017SPaolo Abeni struct mptcp_sock *msk = mptcp_sk(sk); 158580992017SPaolo Abeni 158680992017SPaolo Abeni if (cancel_work_sync(&msk->work)) 158780992017SPaolo Abeni sock_put(sk); 158880992017SPaolo Abeni } 158980992017SPaolo Abeni 159043b54c6eSMat Martineau static void mptcp_subflow_shutdown(struct sock *sk, struct sock *ssk, int how) 159121498490SPeter Krystad { 159221498490SPeter Krystad lock_sock(ssk); 159321498490SPeter Krystad 159421498490SPeter Krystad switch (ssk->sk_state) { 159521498490SPeter Krystad case TCP_LISTEN: 159621498490SPeter Krystad if (!(how & RCV_SHUTDOWN)) 159721498490SPeter Krystad break; 1598df561f66SGustavo A. R. Silva fallthrough; 159921498490SPeter Krystad case TCP_SYN_SENT: 160021498490SPeter Krystad tcp_disconnect(ssk, O_NONBLOCK); 160121498490SPeter Krystad break; 160221498490SPeter Krystad default: 160343b54c6eSMat Martineau if (__mptcp_check_fallback(mptcp_sk(sk))) { 160443b54c6eSMat Martineau pr_debug("Fallback"); 160521498490SPeter Krystad ssk->sk_shutdown |= how; 160621498490SPeter Krystad tcp_shutdown(ssk, how); 160743b54c6eSMat Martineau } else { 160843b54c6eSMat Martineau pr_debug("Sending DATA_FIN on subflow %p", ssk); 160943b54c6eSMat Martineau mptcp_set_timeout(sk, ssk); 161043b54c6eSMat Martineau tcp_send_ack(ssk); 161143b54c6eSMat Martineau } 161221498490SPeter Krystad break; 161321498490SPeter Krystad } 161421498490SPeter Krystad 161521498490SPeter Krystad release_sock(ssk); 161621498490SPeter Krystad } 161721498490SPeter Krystad 16186920b851SMat Martineau static const unsigned char new_state[16] = { 16196920b851SMat Martineau /* current state: new state: action: */ 16206920b851SMat Martineau [0 /* (Invalid) */] = TCP_CLOSE, 16216920b851SMat Martineau [TCP_ESTABLISHED] = TCP_FIN_WAIT1 | TCP_ACTION_FIN, 16226920b851SMat Martineau [TCP_SYN_SENT] = TCP_CLOSE, 16236920b851SMat Martineau [TCP_SYN_RECV] = TCP_FIN_WAIT1 | TCP_ACTION_FIN, 16246920b851SMat Martineau [TCP_FIN_WAIT1] = TCP_FIN_WAIT1, 16256920b851SMat Martineau [TCP_FIN_WAIT2] = TCP_FIN_WAIT2, 16266920b851SMat Martineau [TCP_TIME_WAIT] = TCP_CLOSE, /* should not happen ! */ 16276920b851SMat Martineau [TCP_CLOSE] = TCP_CLOSE, 16286920b851SMat Martineau [TCP_CLOSE_WAIT] = TCP_LAST_ACK | TCP_ACTION_FIN, 16296920b851SMat Martineau [TCP_LAST_ACK] = TCP_LAST_ACK, 16306920b851SMat Martineau [TCP_LISTEN] = TCP_CLOSE, 16316920b851SMat Martineau [TCP_CLOSING] = TCP_CLOSING, 16326920b851SMat Martineau [TCP_NEW_SYN_RECV] = TCP_CLOSE, /* should not happen ! */ 16336920b851SMat Martineau }; 16346920b851SMat Martineau 16356920b851SMat Martineau static int mptcp_close_state(struct sock *sk) 16366920b851SMat Martineau { 16376920b851SMat Martineau int next = (int)new_state[sk->sk_state]; 16386920b851SMat Martineau int ns = next & TCP_STATE_MASK; 16396920b851SMat Martineau 16406920b851SMat Martineau inet_sk_state_store(sk, ns); 16416920b851SMat Martineau 16426920b851SMat Martineau return next & TCP_ACTION_FIN; 16436920b851SMat Martineau } 16446920b851SMat Martineau 16452c22c06cSFlorian Westphal static void mptcp_close(struct sock *sk, long timeout) 1646f870fa0bSMat Martineau { 1647cec37a6eSPeter Krystad struct mptcp_subflow_context *subflow, *tmp; 1648f870fa0bSMat Martineau struct mptcp_sock *msk = mptcp_sk(sk); 1649b2c5b614SFlorian Westphal LIST_HEAD(conn_list); 1650f870fa0bSMat Martineau 16512c22c06cSFlorian Westphal lock_sock(sk); 165243b54c6eSMat Martineau sk->sk_shutdown = SHUTDOWN_MASK; 16532c22c06cSFlorian Westphal 165443b54c6eSMat Martineau if (sk->sk_state == TCP_LISTEN) { 165543b54c6eSMat Martineau inet_sk_state_store(sk, TCP_CLOSE); 165643b54c6eSMat Martineau goto cleanup; 165743b54c6eSMat Martineau } else if (sk->sk_state == TCP_CLOSE) { 165843b54c6eSMat Martineau goto cleanup; 165943b54c6eSMat Martineau } 166043b54c6eSMat Martineau 166143b54c6eSMat Martineau if (__mptcp_check_fallback(msk)) { 166243b54c6eSMat Martineau goto update_state; 166343b54c6eSMat Martineau } else if (mptcp_close_state(sk)) { 166443b54c6eSMat Martineau pr_debug("Sending DATA_FIN sk=%p", sk); 166543b54c6eSMat Martineau WRITE_ONCE(msk->write_seq, msk->write_seq + 1); 166643b54c6eSMat Martineau WRITE_ONCE(msk->snd_data_fin_enable, 1); 166743b54c6eSMat Martineau 166843b54c6eSMat Martineau mptcp_for_each_subflow(msk, subflow) { 166943b54c6eSMat Martineau struct sock *tcp_sk = mptcp_subflow_tcp_sock(subflow); 167043b54c6eSMat Martineau 167143b54c6eSMat Martineau mptcp_subflow_shutdown(sk, tcp_sk, SHUTDOWN_MASK); 167243b54c6eSMat Martineau } 167343b54c6eSMat Martineau } 167443b54c6eSMat Martineau 167543b54c6eSMat Martineau sk_stream_wait_close(sk, timeout); 167643b54c6eSMat Martineau 167743b54c6eSMat Martineau update_state: 1678f870fa0bSMat Martineau inet_sk_state_store(sk, TCP_CLOSE); 1679f870fa0bSMat Martineau 168043b54c6eSMat Martineau cleanup: 168110f6d46cSPaolo Abeni /* be sure to always acquire the join list lock, to sync vs 168210f6d46cSPaolo Abeni * mptcp_finish_join(). 168310f6d46cSPaolo Abeni */ 168410f6d46cSPaolo Abeni spin_lock_bh(&msk->join_list_lock); 168510f6d46cSPaolo Abeni list_splice_tail_init(&msk->join_list, &msk->conn_list); 168610f6d46cSPaolo Abeni spin_unlock_bh(&msk->join_list_lock); 1687b2c5b614SFlorian Westphal list_splice_init(&msk->conn_list, &conn_list); 1688b2c5b614SFlorian Westphal 168918b683bfSPaolo Abeni __mptcp_clear_xmit(sk); 169018b683bfSPaolo Abeni 1691b2c5b614SFlorian Westphal release_sock(sk); 1692b2c5b614SFlorian Westphal 1693b2c5b614SFlorian Westphal list_for_each_entry_safe(subflow, tmp, &conn_list, node) { 1694cec37a6eSPeter Krystad struct sock *ssk = mptcp_subflow_tcp_sock(subflow); 1695cec37a6eSPeter Krystad __mptcp_close_ssk(sk, ssk, subflow, timeout); 1696f870fa0bSMat Martineau } 1697f870fa0bSMat Martineau 169880992017SPaolo Abeni mptcp_cancel_work(sk); 169980992017SPaolo Abeni 17006771bfd9SFlorian Westphal __skb_queue_purge(&sk->sk_receive_queue); 17016771bfd9SFlorian Westphal 1702cec37a6eSPeter Krystad sk_common_release(sk); 1703f870fa0bSMat Martineau } 1704f870fa0bSMat Martineau 1705cf7da0d6SPeter Krystad static void mptcp_copy_inaddrs(struct sock *msk, const struct sock *ssk) 1706cf7da0d6SPeter Krystad { 1707cf7da0d6SPeter Krystad #if IS_ENABLED(CONFIG_MPTCP_IPV6) 1708cf7da0d6SPeter Krystad const struct ipv6_pinfo *ssk6 = inet6_sk(ssk); 1709cf7da0d6SPeter Krystad struct ipv6_pinfo *msk6 = inet6_sk(msk); 1710cf7da0d6SPeter Krystad 1711cf7da0d6SPeter Krystad msk->sk_v6_daddr = ssk->sk_v6_daddr; 1712cf7da0d6SPeter Krystad msk->sk_v6_rcv_saddr = ssk->sk_v6_rcv_saddr; 1713cf7da0d6SPeter Krystad 1714cf7da0d6SPeter Krystad if (msk6 && ssk6) { 1715cf7da0d6SPeter Krystad msk6->saddr = ssk6->saddr; 1716cf7da0d6SPeter Krystad msk6->flow_label = ssk6->flow_label; 1717cf7da0d6SPeter Krystad } 1718cf7da0d6SPeter Krystad #endif 1719cf7da0d6SPeter Krystad 1720cf7da0d6SPeter Krystad inet_sk(msk)->inet_num = inet_sk(ssk)->inet_num; 1721cf7da0d6SPeter Krystad inet_sk(msk)->inet_dport = inet_sk(ssk)->inet_dport; 1722cf7da0d6SPeter Krystad inet_sk(msk)->inet_sport = inet_sk(ssk)->inet_sport; 1723cf7da0d6SPeter Krystad inet_sk(msk)->inet_daddr = inet_sk(ssk)->inet_daddr; 1724cf7da0d6SPeter Krystad inet_sk(msk)->inet_saddr = inet_sk(ssk)->inet_saddr; 1725cf7da0d6SPeter Krystad inet_sk(msk)->inet_rcv_saddr = inet_sk(ssk)->inet_rcv_saddr; 1726cf7da0d6SPeter Krystad } 1727cf7da0d6SPeter Krystad 172818b683bfSPaolo Abeni static int mptcp_disconnect(struct sock *sk, int flags) 172918b683bfSPaolo Abeni { 173042c556feSFlorian Westphal /* Should never be called. 173142c556feSFlorian Westphal * inet_stream_connect() calls ->disconnect, but that 173242c556feSFlorian Westphal * refers to the subflow socket, not the mptcp one. 173342c556feSFlorian Westphal */ 173442c556feSFlorian Westphal WARN_ON_ONCE(1); 173542c556feSFlorian Westphal return 0; 173618b683bfSPaolo Abeni } 173718b683bfSPaolo Abeni 1738b0519de8SFlorian Westphal #if IS_ENABLED(CONFIG_MPTCP_IPV6) 1739b0519de8SFlorian Westphal static struct ipv6_pinfo *mptcp_inet6_sk(const struct sock *sk) 1740b0519de8SFlorian Westphal { 1741b0519de8SFlorian Westphal unsigned int offset = sizeof(struct mptcp6_sock) - sizeof(struct ipv6_pinfo); 1742b0519de8SFlorian Westphal 1743b0519de8SFlorian Westphal return (struct ipv6_pinfo *)(((u8 *)sk) + offset); 1744b0519de8SFlorian Westphal } 1745b0519de8SFlorian Westphal #endif 1746b0519de8SFlorian Westphal 1747fca5c82cSPaolo Abeni struct sock *mptcp_sk_clone(const struct sock *sk, 1748cfde141eSPaolo Abeni const struct mptcp_options_received *mp_opt, 1749fca5c82cSPaolo Abeni struct request_sock *req) 1750b0519de8SFlorian Westphal { 175158b09919SPaolo Abeni struct mptcp_subflow_request_sock *subflow_req = mptcp_subflow_rsk(req); 1752b0519de8SFlorian Westphal struct sock *nsk = sk_clone_lock(sk, GFP_ATOMIC); 175358b09919SPaolo Abeni struct mptcp_sock *msk; 175458b09919SPaolo Abeni u64 ack_seq; 1755b0519de8SFlorian Westphal 1756b0519de8SFlorian Westphal if (!nsk) 1757b0519de8SFlorian Westphal return NULL; 1758b0519de8SFlorian Westphal 1759b0519de8SFlorian Westphal #if IS_ENABLED(CONFIG_MPTCP_IPV6) 1760b0519de8SFlorian Westphal if (nsk->sk_family == AF_INET6) 1761b0519de8SFlorian Westphal inet_sk(nsk)->pinet6 = mptcp_inet6_sk(nsk); 1762b0519de8SFlorian Westphal #endif 1763b0519de8SFlorian Westphal 176458b09919SPaolo Abeni __mptcp_init_sock(nsk); 176558b09919SPaolo Abeni 176658b09919SPaolo Abeni msk = mptcp_sk(nsk); 176758b09919SPaolo Abeni msk->local_key = subflow_req->local_key; 176858b09919SPaolo Abeni msk->token = subflow_req->token; 176958b09919SPaolo Abeni msk->subflow = NULL; 1770b93df08cSPaolo Abeni WRITE_ONCE(msk->fully_established, false); 177158b09919SPaolo Abeni 177258b09919SPaolo Abeni msk->write_seq = subflow_req->idsn + 1; 1773cc9d2566SPaolo Abeni atomic64_set(&msk->snd_una, msk->write_seq); 1774cfde141eSPaolo Abeni if (mp_opt->mp_capable) { 177558b09919SPaolo Abeni msk->can_ack = true; 1776cfde141eSPaolo Abeni msk->remote_key = mp_opt->sndr_key; 177758b09919SPaolo Abeni mptcp_crypto_key_sha(msk->remote_key, NULL, &ack_seq); 177858b09919SPaolo Abeni ack_seq++; 177958b09919SPaolo Abeni msk->ack_seq = ack_seq; 178058b09919SPaolo Abeni } 17817f20d5fcSPaolo Abeni 17825e20087dSFlorian Westphal sock_reset_flag(nsk, SOCK_RCU_FREE); 17837f20d5fcSPaolo Abeni /* will be fully established after successful MPC subflow creation */ 17847f20d5fcSPaolo Abeni inet_sk_state_store(nsk, TCP_SYN_RECV); 178558b09919SPaolo Abeni bh_unlock_sock(nsk); 178658b09919SPaolo Abeni 178758b09919SPaolo Abeni /* keep a single reference */ 178858b09919SPaolo Abeni __sock_put(nsk); 1789b0519de8SFlorian Westphal return nsk; 1790b0519de8SFlorian Westphal } 1791b0519de8SFlorian Westphal 1792a6b118feSFlorian Westphal void mptcp_rcv_space_init(struct mptcp_sock *msk, const struct sock *ssk) 1793a6b118feSFlorian Westphal { 1794a6b118feSFlorian Westphal const struct tcp_sock *tp = tcp_sk(ssk); 1795a6b118feSFlorian Westphal 1796a6b118feSFlorian Westphal msk->rcvq_space.copied = 0; 1797a6b118feSFlorian Westphal msk->rcvq_space.rtt_us = 0; 1798a6b118feSFlorian Westphal 1799a6b118feSFlorian Westphal msk->rcvq_space.time = tp->tcp_mstamp; 1800a6b118feSFlorian Westphal 1801a6b118feSFlorian Westphal /* initial rcv_space offering made to peer */ 1802a6b118feSFlorian Westphal msk->rcvq_space.space = min_t(u32, tp->rcv_wnd, 1803a6b118feSFlorian Westphal TCP_INIT_CWND * tp->advmss); 1804a6b118feSFlorian Westphal if (msk->rcvq_space.space == 0) 1805a6b118feSFlorian Westphal msk->rcvq_space.space = TCP_INIT_CWND * TCP_MSS_DEFAULT; 1806a6b118feSFlorian Westphal } 1807a6b118feSFlorian Westphal 1808cf7da0d6SPeter Krystad static struct sock *mptcp_accept(struct sock *sk, int flags, int *err, 1809cf7da0d6SPeter Krystad bool kern) 1810cf7da0d6SPeter Krystad { 1811cf7da0d6SPeter Krystad struct mptcp_sock *msk = mptcp_sk(sk); 1812cf7da0d6SPeter Krystad struct socket *listener; 1813cf7da0d6SPeter Krystad struct sock *newsk; 1814cf7da0d6SPeter Krystad 1815cf7da0d6SPeter Krystad listener = __mptcp_nmpc_socket(msk); 1816cf7da0d6SPeter Krystad if (WARN_ON_ONCE(!listener)) { 1817cf7da0d6SPeter Krystad *err = -EINVAL; 1818cf7da0d6SPeter Krystad return NULL; 1819cf7da0d6SPeter Krystad } 1820cf7da0d6SPeter Krystad 1821cf7da0d6SPeter Krystad pr_debug("msk=%p, listener=%p", msk, mptcp_subflow_ctx(listener->sk)); 1822cf7da0d6SPeter Krystad newsk = inet_csk_accept(listener->sk, flags, err, kern); 1823cf7da0d6SPeter Krystad if (!newsk) 1824cf7da0d6SPeter Krystad return NULL; 1825cf7da0d6SPeter Krystad 1826cf7da0d6SPeter Krystad pr_debug("msk=%p, subflow is mptcp=%d", msk, sk_is_mptcp(newsk)); 1827cf7da0d6SPeter Krystad if (sk_is_mptcp(newsk)) { 1828cf7da0d6SPeter Krystad struct mptcp_subflow_context *subflow; 1829cf7da0d6SPeter Krystad struct sock *new_mptcp_sock; 1830cf7da0d6SPeter Krystad struct sock *ssk = newsk; 1831cf7da0d6SPeter Krystad 1832cf7da0d6SPeter Krystad subflow = mptcp_subflow_ctx(newsk); 183358b09919SPaolo Abeni new_mptcp_sock = subflow->conn; 183458b09919SPaolo Abeni 183558b09919SPaolo Abeni /* is_mptcp should be false if subflow->conn is missing, see 183658b09919SPaolo Abeni * subflow_syn_recv_sock() 183758b09919SPaolo Abeni */ 183858b09919SPaolo Abeni if (WARN_ON_ONCE(!new_mptcp_sock)) { 183958b09919SPaolo Abeni tcp_sk(newsk)->is_mptcp = 0; 184058b09919SPaolo Abeni return newsk; 184158b09919SPaolo Abeni } 184258b09919SPaolo Abeni 184358b09919SPaolo Abeni /* acquire the 2nd reference for the owning socket */ 184458b09919SPaolo Abeni sock_hold(new_mptcp_sock); 1845cf7da0d6SPeter Krystad 1846cf7da0d6SPeter Krystad local_bh_disable(); 184758b09919SPaolo Abeni bh_lock_sock(new_mptcp_sock); 1848cf7da0d6SPeter Krystad msk = mptcp_sk(new_mptcp_sock); 18498ab183deSPaolo Abeni msk->first = newsk; 1850cf7da0d6SPeter Krystad 1851cf7da0d6SPeter Krystad newsk = new_mptcp_sock; 1852cf7da0d6SPeter Krystad mptcp_copy_inaddrs(newsk, ssk); 1853cf7da0d6SPeter Krystad list_add(&subflow->node, &msk->conn_list); 1854cf7da0d6SPeter Krystad 1855a6b118feSFlorian Westphal mptcp_rcv_space_init(msk, ssk); 1856cf7da0d6SPeter Krystad bh_unlock_sock(new_mptcp_sock); 1857fc518953SFlorian Westphal 1858fc518953SFlorian Westphal __MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_MPCAPABLEPASSIVEACK); 1859cf7da0d6SPeter Krystad local_bh_enable(); 1860fc518953SFlorian Westphal } else { 1861fc518953SFlorian Westphal MPTCP_INC_STATS(sock_net(sk), 1862fc518953SFlorian Westphal MPTCP_MIB_MPCAPABLEPASSIVEFALLBACK); 1863cf7da0d6SPeter Krystad } 1864cf7da0d6SPeter Krystad 1865cf7da0d6SPeter Krystad return newsk; 1866cf7da0d6SPeter Krystad } 1867cf7da0d6SPeter Krystad 186879c0949eSPeter Krystad static void mptcp_destroy(struct sock *sk) 186979c0949eSPeter Krystad { 1870c9fd9c5fSFlorian Westphal struct mptcp_sock *msk = mptcp_sk(sk); 1871c9fd9c5fSFlorian Westphal 18722c5ebd00SPaolo Abeni mptcp_token_destroy(msk); 1873c9fd9c5fSFlorian Westphal if (msk->cached_ext) 1874c9fd9c5fSFlorian Westphal __skb_ext_put(msk->cached_ext); 1875d027236cSPaolo Abeni 1876d027236cSPaolo Abeni sk_sockets_allocated_dec(sk); 187779c0949eSPeter Krystad } 187879c0949eSPeter Krystad 1879fd1452d8SFlorian Westphal static int mptcp_setsockopt_sol_socket(struct mptcp_sock *msk, int optname, 1880a7b75c5aSChristoph Hellwig sockptr_t optval, unsigned int optlen) 1881fd1452d8SFlorian Westphal { 1882fd1452d8SFlorian Westphal struct sock *sk = (struct sock *)msk; 1883fd1452d8SFlorian Westphal struct socket *ssock; 1884fd1452d8SFlorian Westphal int ret; 1885fd1452d8SFlorian Westphal 1886fd1452d8SFlorian Westphal switch (optname) { 1887fd1452d8SFlorian Westphal case SO_REUSEPORT: 1888fd1452d8SFlorian Westphal case SO_REUSEADDR: 1889fd1452d8SFlorian Westphal lock_sock(sk); 1890fd1452d8SFlorian Westphal ssock = __mptcp_nmpc_socket(msk); 1891fd1452d8SFlorian Westphal if (!ssock) { 1892fd1452d8SFlorian Westphal release_sock(sk); 1893fd1452d8SFlorian Westphal return -EINVAL; 1894fd1452d8SFlorian Westphal } 1895fd1452d8SFlorian Westphal 1896a7b75c5aSChristoph Hellwig ret = sock_setsockopt(ssock, SOL_SOCKET, optname, optval, optlen); 1897fd1452d8SFlorian Westphal if (ret == 0) { 1898fd1452d8SFlorian Westphal if (optname == SO_REUSEPORT) 1899fd1452d8SFlorian Westphal sk->sk_reuseport = ssock->sk->sk_reuseport; 1900fd1452d8SFlorian Westphal else if (optname == SO_REUSEADDR) 1901fd1452d8SFlorian Westphal sk->sk_reuse = ssock->sk->sk_reuse; 1902fd1452d8SFlorian Westphal } 1903fd1452d8SFlorian Westphal release_sock(sk); 1904fd1452d8SFlorian Westphal return ret; 1905fd1452d8SFlorian Westphal } 1906fd1452d8SFlorian Westphal 1907a7b75c5aSChristoph Hellwig return sock_setsockopt(sk->sk_socket, SOL_SOCKET, optname, optval, optlen); 1908fd1452d8SFlorian Westphal } 1909fd1452d8SFlorian Westphal 1910c9b95a13SFlorian Westphal static int mptcp_setsockopt_v6(struct mptcp_sock *msk, int optname, 1911a7b75c5aSChristoph Hellwig sockptr_t optval, unsigned int optlen) 1912c9b95a13SFlorian Westphal { 1913c9b95a13SFlorian Westphal struct sock *sk = (struct sock *)msk; 1914c9b95a13SFlorian Westphal int ret = -EOPNOTSUPP; 1915c9b95a13SFlorian Westphal struct socket *ssock; 1916c9b95a13SFlorian Westphal 1917c9b95a13SFlorian Westphal switch (optname) { 1918c9b95a13SFlorian Westphal case IPV6_V6ONLY: 1919c9b95a13SFlorian Westphal lock_sock(sk); 1920c9b95a13SFlorian Westphal ssock = __mptcp_nmpc_socket(msk); 1921c9b95a13SFlorian Westphal if (!ssock) { 1922c9b95a13SFlorian Westphal release_sock(sk); 1923c9b95a13SFlorian Westphal return -EINVAL; 1924c9b95a13SFlorian Westphal } 1925c9b95a13SFlorian Westphal 1926c9b95a13SFlorian Westphal ret = tcp_setsockopt(ssock->sk, SOL_IPV6, optname, optval, optlen); 1927c9b95a13SFlorian Westphal if (ret == 0) 1928c9b95a13SFlorian Westphal sk->sk_ipv6only = ssock->sk->sk_ipv6only; 1929c9b95a13SFlorian Westphal 1930c9b95a13SFlorian Westphal release_sock(sk); 1931c9b95a13SFlorian Westphal break; 1932c9b95a13SFlorian Westphal } 1933c9b95a13SFlorian Westphal 1934c9b95a13SFlorian Westphal return ret; 1935c9b95a13SFlorian Westphal } 1936c9b95a13SFlorian Westphal 1937717e79c8SPeter Krystad static int mptcp_setsockopt(struct sock *sk, int level, int optname, 1938a7b75c5aSChristoph Hellwig sockptr_t optval, unsigned int optlen) 1939717e79c8SPeter Krystad { 1940717e79c8SPeter Krystad struct mptcp_sock *msk = mptcp_sk(sk); 194176660afbSPaolo Abeni struct sock *ssk; 1942717e79c8SPeter Krystad 1943717e79c8SPeter Krystad pr_debug("msk=%p", msk); 1944717e79c8SPeter Krystad 194583f0c10bSFlorian Westphal if (level == SOL_SOCKET) 1946fd1452d8SFlorian Westphal return mptcp_setsockopt_sol_socket(msk, optname, optval, optlen); 194783f0c10bSFlorian Westphal 1948717e79c8SPeter Krystad /* @@ the meaning of setsockopt() when the socket is connected and 1949b6e4a1aeSMat Martineau * there are multiple subflows is not yet defined. It is up to the 1950b6e4a1aeSMat Martineau * MPTCP-level socket to configure the subflows until the subflow 1951b6e4a1aeSMat Martineau * is in TCP fallback, when TCP socket options are passed through 1952b6e4a1aeSMat Martineau * to the one remaining subflow. 1953717e79c8SPeter Krystad */ 1954717e79c8SPeter Krystad lock_sock(sk); 195576660afbSPaolo Abeni ssk = __mptcp_tcp_fallback(msk); 1956e154659bSFlorian Westphal release_sock(sk); 195776660afbSPaolo Abeni if (ssk) 195876660afbSPaolo Abeni return tcp_setsockopt(ssk, level, optname, optval, optlen); 195950e741bbSFlorian Westphal 1960c9b95a13SFlorian Westphal if (level == SOL_IPV6) 1961c9b95a13SFlorian Westphal return mptcp_setsockopt_v6(msk, optname, optval, optlen); 1962c9b95a13SFlorian Westphal 1963b6e4a1aeSMat Martineau return -EOPNOTSUPP; 1964717e79c8SPeter Krystad } 1965717e79c8SPeter Krystad 1966717e79c8SPeter Krystad static int mptcp_getsockopt(struct sock *sk, int level, int optname, 196750e741bbSFlorian Westphal char __user *optval, int __user *option) 1968717e79c8SPeter Krystad { 1969717e79c8SPeter Krystad struct mptcp_sock *msk = mptcp_sk(sk); 197076660afbSPaolo Abeni struct sock *ssk; 1971717e79c8SPeter Krystad 1972717e79c8SPeter Krystad pr_debug("msk=%p", msk); 1973717e79c8SPeter Krystad 1974b6e4a1aeSMat Martineau /* @@ the meaning of setsockopt() when the socket is connected and 1975b6e4a1aeSMat Martineau * there are multiple subflows is not yet defined. It is up to the 1976b6e4a1aeSMat Martineau * MPTCP-level socket to configure the subflows until the subflow 1977b6e4a1aeSMat Martineau * is in TCP fallback, when socket options are passed through 1978b6e4a1aeSMat Martineau * to the one remaining subflow. 1979717e79c8SPeter Krystad */ 1980717e79c8SPeter Krystad lock_sock(sk); 198176660afbSPaolo Abeni ssk = __mptcp_tcp_fallback(msk); 1982e154659bSFlorian Westphal release_sock(sk); 198376660afbSPaolo Abeni if (ssk) 198476660afbSPaolo Abeni return tcp_getsockopt(ssk, level, optname, optval, option); 198550e741bbSFlorian Westphal 1986b6e4a1aeSMat Martineau return -EOPNOTSUPP; 1987717e79c8SPeter Krystad } 1988717e79c8SPeter Krystad 1989b51f9b80SPaolo Abeni #define MPTCP_DEFERRED_ALL (TCPF_DELACK_TIMER_DEFERRED | \ 1990b51f9b80SPaolo Abeni TCPF_WRITE_TIMER_DEFERRED) 199114c441b5SPaolo Abeni 199214c441b5SPaolo Abeni /* this is very alike tcp_release_cb() but we must handle differently a 199314c441b5SPaolo Abeni * different set of events 199414c441b5SPaolo Abeni */ 199514c441b5SPaolo Abeni static void mptcp_release_cb(struct sock *sk) 199614c441b5SPaolo Abeni { 199714c441b5SPaolo Abeni unsigned long flags, nflags; 199814c441b5SPaolo Abeni 199914c441b5SPaolo Abeni do { 200014c441b5SPaolo Abeni flags = sk->sk_tsq_flags; 200114c441b5SPaolo Abeni if (!(flags & MPTCP_DEFERRED_ALL)) 200214c441b5SPaolo Abeni return; 200314c441b5SPaolo Abeni nflags = flags & ~MPTCP_DEFERRED_ALL; 200414c441b5SPaolo Abeni } while (cmpxchg(&sk->sk_tsq_flags, flags, nflags) != flags); 200514c441b5SPaolo Abeni 2006b51f9b80SPaolo Abeni sock_release_ownership(sk); 2007b51f9b80SPaolo Abeni 200814c441b5SPaolo Abeni if (flags & TCPF_DELACK_TIMER_DEFERRED) { 200914c441b5SPaolo Abeni struct mptcp_sock *msk = mptcp_sk(sk); 201014c441b5SPaolo Abeni struct sock *ssk; 201114c441b5SPaolo Abeni 201214c441b5SPaolo Abeni ssk = mptcp_subflow_recv_lookup(msk); 201314c441b5SPaolo Abeni if (!ssk || !schedule_work(&msk->work)) 201414c441b5SPaolo Abeni __sock_put(sk); 201514c441b5SPaolo Abeni } 2016b51f9b80SPaolo Abeni 2017b51f9b80SPaolo Abeni if (flags & TCPF_WRITE_TIMER_DEFERRED) { 2018b51f9b80SPaolo Abeni mptcp_retransmit_handler(sk); 2019b51f9b80SPaolo Abeni __sock_put(sk); 2020b51f9b80SPaolo Abeni } 202114c441b5SPaolo Abeni } 202214c441b5SPaolo Abeni 20232c5ebd00SPaolo Abeni static int mptcp_hash(struct sock *sk) 20242c5ebd00SPaolo Abeni { 20252c5ebd00SPaolo Abeni /* should never be called, 20262c5ebd00SPaolo Abeni * we hash the TCP subflows not the master socket 20272c5ebd00SPaolo Abeni */ 20282c5ebd00SPaolo Abeni WARN_ON_ONCE(1); 20292c5ebd00SPaolo Abeni return 0; 20302c5ebd00SPaolo Abeni } 20312c5ebd00SPaolo Abeni 20322c5ebd00SPaolo Abeni static void mptcp_unhash(struct sock *sk) 20332c5ebd00SPaolo Abeni { 20342c5ebd00SPaolo Abeni /* called from sk_common_release(), but nothing to do here */ 20352c5ebd00SPaolo Abeni } 20362c5ebd00SPaolo Abeni 2037cec37a6eSPeter Krystad static int mptcp_get_port(struct sock *sk, unsigned short snum) 2038f870fa0bSMat Martineau { 2039f870fa0bSMat Martineau struct mptcp_sock *msk = mptcp_sk(sk); 2040cec37a6eSPeter Krystad struct socket *ssock; 2041f870fa0bSMat Martineau 2042cec37a6eSPeter Krystad ssock = __mptcp_nmpc_socket(msk); 2043cec37a6eSPeter Krystad pr_debug("msk=%p, subflow=%p", msk, ssock); 2044cec37a6eSPeter Krystad if (WARN_ON_ONCE(!ssock)) 2045cec37a6eSPeter Krystad return -EINVAL; 2046f870fa0bSMat Martineau 2047cec37a6eSPeter Krystad return inet_csk_get_port(ssock->sk, snum); 2048cec37a6eSPeter Krystad } 2049f870fa0bSMat Martineau 2050cec37a6eSPeter Krystad void mptcp_finish_connect(struct sock *ssk) 2051cec37a6eSPeter Krystad { 2052cec37a6eSPeter Krystad struct mptcp_subflow_context *subflow; 2053cec37a6eSPeter Krystad struct mptcp_sock *msk; 2054cec37a6eSPeter Krystad struct sock *sk; 20556d0060f6SMat Martineau u64 ack_seq; 2056f870fa0bSMat Martineau 2057cec37a6eSPeter Krystad subflow = mptcp_subflow_ctx(ssk); 2058cec37a6eSPeter Krystad sk = subflow->conn; 2059cec37a6eSPeter Krystad msk = mptcp_sk(sk); 2060cec37a6eSPeter Krystad 2061648ef4b8SMat Martineau pr_debug("msk=%p, token=%u", sk, subflow->token); 2062648ef4b8SMat Martineau 20636d0060f6SMat Martineau mptcp_crypto_key_sha(subflow->remote_key, NULL, &ack_seq); 20646d0060f6SMat Martineau ack_seq++; 2065648ef4b8SMat Martineau subflow->map_seq = ack_seq; 2066648ef4b8SMat Martineau subflow->map_subflow_seq = 1; 20676d0060f6SMat Martineau 2068cec37a6eSPeter Krystad /* the socket is not connected yet, no msk/subflow ops can access/race 2069cec37a6eSPeter Krystad * accessing the field below 2070cec37a6eSPeter Krystad */ 2071cec37a6eSPeter Krystad WRITE_ONCE(msk->remote_key, subflow->remote_key); 2072cec37a6eSPeter Krystad WRITE_ONCE(msk->local_key, subflow->local_key); 20736d0060f6SMat Martineau WRITE_ONCE(msk->write_seq, subflow->idsn + 1); 20746d0060f6SMat Martineau WRITE_ONCE(msk->ack_seq, ack_seq); 2075d22f4988SChristoph Paasch WRITE_ONCE(msk->can_ack, 1); 2076cc9d2566SPaolo Abeni atomic64_set(&msk->snd_una, msk->write_seq); 20771b1c7a0eSPeter Krystad 20781b1c7a0eSPeter Krystad mptcp_pm_new_connection(msk, 0); 2079a6b118feSFlorian Westphal 2080a6b118feSFlorian Westphal mptcp_rcv_space_init(msk, ssk); 2081f870fa0bSMat Martineau } 2082f870fa0bSMat Martineau 2083cf7da0d6SPeter Krystad static void mptcp_sock_graft(struct sock *sk, struct socket *parent) 2084cf7da0d6SPeter Krystad { 2085cf7da0d6SPeter Krystad write_lock_bh(&sk->sk_callback_lock); 2086cf7da0d6SPeter Krystad rcu_assign_pointer(sk->sk_wq, &parent->wq); 2087cf7da0d6SPeter Krystad sk_set_socket(sk, parent); 2088cf7da0d6SPeter Krystad sk->sk_uid = SOCK_INODE(parent)->i_uid; 2089cf7da0d6SPeter Krystad write_unlock_bh(&sk->sk_callback_lock); 2090cf7da0d6SPeter Krystad } 2091cf7da0d6SPeter Krystad 2092f296234cSPeter Krystad bool mptcp_finish_join(struct sock *sk) 2093f296234cSPeter Krystad { 2094f296234cSPeter Krystad struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(sk); 2095f296234cSPeter Krystad struct mptcp_sock *msk = mptcp_sk(subflow->conn); 2096f296234cSPeter Krystad struct sock *parent = (void *)msk; 2097f296234cSPeter Krystad struct socket *parent_sock; 2098ec3edaa7SPeter Krystad bool ret; 2099f296234cSPeter Krystad 2100f296234cSPeter Krystad pr_debug("msk=%p, subflow=%p", msk, subflow); 2101f296234cSPeter Krystad 2102f296234cSPeter Krystad /* mptcp socket already closing? */ 2103b93df08cSPaolo Abeni if (!mptcp_is_fully_established(parent)) 2104f296234cSPeter Krystad return false; 2105f296234cSPeter Krystad 2106f296234cSPeter Krystad if (!msk->pm.server_side) 2107f296234cSPeter Krystad return true; 2108f296234cSPeter Krystad 210910f6d46cSPaolo Abeni if (!mptcp_pm_allow_new_subflow(msk)) 211010f6d46cSPaolo Abeni return false; 211110f6d46cSPaolo Abeni 211210f6d46cSPaolo Abeni /* active connections are already on conn_list, and we can't acquire 211310f6d46cSPaolo Abeni * msk lock here. 211410f6d46cSPaolo Abeni * use the join list lock as synchronization point and double-check 211510f6d46cSPaolo Abeni * msk status to avoid racing with mptcp_close() 211610f6d46cSPaolo Abeni */ 211710f6d46cSPaolo Abeni spin_lock_bh(&msk->join_list_lock); 211810f6d46cSPaolo Abeni ret = inet_sk_state_load(parent) == TCP_ESTABLISHED; 211910f6d46cSPaolo Abeni if (ret && !WARN_ON_ONCE(!list_empty(&subflow->node))) 212010f6d46cSPaolo Abeni list_add_tail(&subflow->node, &msk->join_list); 212110f6d46cSPaolo Abeni spin_unlock_bh(&msk->join_list_lock); 212210f6d46cSPaolo Abeni if (!ret) 212310f6d46cSPaolo Abeni return false; 212410f6d46cSPaolo Abeni 212510f6d46cSPaolo Abeni /* attach to msk socket only after we are sure he will deal with us 212610f6d46cSPaolo Abeni * at close time 212710f6d46cSPaolo Abeni */ 2128f296234cSPeter Krystad parent_sock = READ_ONCE(parent->sk_socket); 2129f296234cSPeter Krystad if (parent_sock && !sk->sk_socket) 2130f296234cSPeter Krystad mptcp_sock_graft(sk, parent_sock); 213164d950aeSChristoph Paasch subflow->map_seq = msk->ack_seq; 213210f6d46cSPaolo Abeni return true; 2133f296234cSPeter Krystad } 2134f296234cSPeter Krystad 21351891c4a0SFlorian Westphal static bool mptcp_memory_free(const struct sock *sk, int wake) 21361891c4a0SFlorian Westphal { 21371891c4a0SFlorian Westphal struct mptcp_sock *msk = mptcp_sk(sk); 21381891c4a0SFlorian Westphal 21391891c4a0SFlorian Westphal return wake ? test_bit(MPTCP_SEND_SPACE, &msk->flags) : true; 21401891c4a0SFlorian Westphal } 21411891c4a0SFlorian Westphal 2142f870fa0bSMat Martineau static struct proto mptcp_prot = { 2143f870fa0bSMat Martineau .name = "MPTCP", 2144f870fa0bSMat Martineau .owner = THIS_MODULE, 2145f870fa0bSMat Martineau .init = mptcp_init_sock, 214618b683bfSPaolo Abeni .disconnect = mptcp_disconnect, 2147f870fa0bSMat Martineau .close = mptcp_close, 2148cf7da0d6SPeter Krystad .accept = mptcp_accept, 2149717e79c8SPeter Krystad .setsockopt = mptcp_setsockopt, 2150717e79c8SPeter Krystad .getsockopt = mptcp_getsockopt, 2151f870fa0bSMat Martineau .shutdown = tcp_shutdown, 215279c0949eSPeter Krystad .destroy = mptcp_destroy, 2153f870fa0bSMat Martineau .sendmsg = mptcp_sendmsg, 2154f870fa0bSMat Martineau .recvmsg = mptcp_recvmsg, 215514c441b5SPaolo Abeni .release_cb = mptcp_release_cb, 21562c5ebd00SPaolo Abeni .hash = mptcp_hash, 21572c5ebd00SPaolo Abeni .unhash = mptcp_unhash, 2158cec37a6eSPeter Krystad .get_port = mptcp_get_port, 2159d027236cSPaolo Abeni .sockets_allocated = &mptcp_sockets_allocated, 2160d027236cSPaolo Abeni .memory_allocated = &tcp_memory_allocated, 2161d027236cSPaolo Abeni .memory_pressure = &tcp_memory_pressure, 21621891c4a0SFlorian Westphal .stream_memory_free = mptcp_memory_free, 2163d027236cSPaolo Abeni .sysctl_wmem_offset = offsetof(struct net, ipv4.sysctl_tcp_wmem), 2164d027236cSPaolo Abeni .sysctl_mem = sysctl_tcp_mem, 2165f870fa0bSMat Martineau .obj_size = sizeof(struct mptcp_sock), 21662c5ebd00SPaolo Abeni .slab_flags = SLAB_TYPESAFE_BY_RCU, 2167f870fa0bSMat Martineau .no_autobind = true, 2168f870fa0bSMat Martineau }; 2169f870fa0bSMat Martineau 21702303f994SPeter Krystad static int mptcp_bind(struct socket *sock, struct sockaddr *uaddr, int addr_len) 21712303f994SPeter Krystad { 21722303f994SPeter Krystad struct mptcp_sock *msk = mptcp_sk(sock->sk); 21732303f994SPeter Krystad struct socket *ssock; 2174cf7da0d6SPeter Krystad int err; 21752303f994SPeter Krystad 21762303f994SPeter Krystad lock_sock(sock->sk); 2177fa68018dSPaolo Abeni ssock = __mptcp_nmpc_socket(msk); 2178fa68018dSPaolo Abeni if (!ssock) { 2179fa68018dSPaolo Abeni err = -EINVAL; 21802303f994SPeter Krystad goto unlock; 21812303f994SPeter Krystad } 21822303f994SPeter Krystad 21832303f994SPeter Krystad err = ssock->ops->bind(ssock, uaddr, addr_len); 2184cf7da0d6SPeter Krystad if (!err) 2185cf7da0d6SPeter Krystad mptcp_copy_inaddrs(sock->sk, ssock->sk); 21862303f994SPeter Krystad 21872303f994SPeter Krystad unlock: 21882303f994SPeter Krystad release_sock(sock->sk); 21892303f994SPeter Krystad return err; 21902303f994SPeter Krystad } 21912303f994SPeter Krystad 21920235d075SPaolo Abeni static void mptcp_subflow_early_fallback(struct mptcp_sock *msk, 21930235d075SPaolo Abeni struct mptcp_subflow_context *subflow) 21940235d075SPaolo Abeni { 21950235d075SPaolo Abeni subflow->request_mptcp = 0; 21960235d075SPaolo Abeni __mptcp_do_fallback(msk); 21970235d075SPaolo Abeni } 21980235d075SPaolo Abeni 21992303f994SPeter Krystad static int mptcp_stream_connect(struct socket *sock, struct sockaddr *uaddr, 22002303f994SPeter Krystad int addr_len, int flags) 22012303f994SPeter Krystad { 22022303f994SPeter Krystad struct mptcp_sock *msk = mptcp_sk(sock->sk); 22032c5ebd00SPaolo Abeni struct mptcp_subflow_context *subflow; 22042303f994SPeter Krystad struct socket *ssock; 22052303f994SPeter Krystad int err; 22062303f994SPeter Krystad 22072303f994SPeter Krystad lock_sock(sock->sk); 220841be81a8SPaolo Abeni if (sock->state != SS_UNCONNECTED && msk->subflow) { 220941be81a8SPaolo Abeni /* pending connection or invalid state, let existing subflow 221041be81a8SPaolo Abeni * cope with that 221141be81a8SPaolo Abeni */ 221241be81a8SPaolo Abeni ssock = msk->subflow; 221341be81a8SPaolo Abeni goto do_connect; 221441be81a8SPaolo Abeni } 221541be81a8SPaolo Abeni 2216fa68018dSPaolo Abeni ssock = __mptcp_nmpc_socket(msk); 2217fa68018dSPaolo Abeni if (!ssock) { 2218fa68018dSPaolo Abeni err = -EINVAL; 22192303f994SPeter Krystad goto unlock; 22202303f994SPeter Krystad } 22212303f994SPeter Krystad 2222fa68018dSPaolo Abeni mptcp_token_destroy(msk); 2223fa68018dSPaolo Abeni inet_sk_state_store(sock->sk, TCP_SYN_SENT); 22242c5ebd00SPaolo Abeni subflow = mptcp_subflow_ctx(ssock->sk); 2225cf7da0d6SPeter Krystad #ifdef CONFIG_TCP_MD5SIG 2226cf7da0d6SPeter Krystad /* no MPTCP if MD5SIG is enabled on this socket or we may run out of 2227cf7da0d6SPeter Krystad * TCP option space. 2228cf7da0d6SPeter Krystad */ 2229cf7da0d6SPeter Krystad if (rcu_access_pointer(tcp_sk(ssock->sk)->md5sig_info)) 22300235d075SPaolo Abeni mptcp_subflow_early_fallback(msk, subflow); 2231cf7da0d6SPeter Krystad #endif 22322c5ebd00SPaolo Abeni if (subflow->request_mptcp && mptcp_token_new_connect(ssock->sk)) 22330235d075SPaolo Abeni mptcp_subflow_early_fallback(msk, subflow); 2234cf7da0d6SPeter Krystad 223541be81a8SPaolo Abeni do_connect: 22362303f994SPeter Krystad err = ssock->ops->connect(ssock, uaddr, addr_len, flags); 223741be81a8SPaolo Abeni sock->state = ssock->state; 223841be81a8SPaolo Abeni 223941be81a8SPaolo Abeni /* on successful connect, the msk state will be moved to established by 224041be81a8SPaolo Abeni * subflow_finish_connect() 224141be81a8SPaolo Abeni */ 2242367fe04eSMatthieu Baerts if (!err || err == -EINPROGRESS) 2243cf7da0d6SPeter Krystad mptcp_copy_inaddrs(sock->sk, ssock->sk); 224441be81a8SPaolo Abeni else 224541be81a8SPaolo Abeni inet_sk_state_store(sock->sk, inet_sk_state_load(ssock->sk)); 22462303f994SPeter Krystad 22472303f994SPeter Krystad unlock: 22482303f994SPeter Krystad release_sock(sock->sk); 22492303f994SPeter Krystad return err; 22502303f994SPeter Krystad } 22512303f994SPeter Krystad 2252cf7da0d6SPeter Krystad static int mptcp_listen(struct socket *sock, int backlog) 2253cf7da0d6SPeter Krystad { 2254cf7da0d6SPeter Krystad struct mptcp_sock *msk = mptcp_sk(sock->sk); 2255cf7da0d6SPeter Krystad struct socket *ssock; 2256cf7da0d6SPeter Krystad int err; 2257cf7da0d6SPeter Krystad 2258cf7da0d6SPeter Krystad pr_debug("msk=%p", msk); 2259cf7da0d6SPeter Krystad 2260cf7da0d6SPeter Krystad lock_sock(sock->sk); 2261fa68018dSPaolo Abeni ssock = __mptcp_nmpc_socket(msk); 2262fa68018dSPaolo Abeni if (!ssock) { 2263fa68018dSPaolo Abeni err = -EINVAL; 2264cf7da0d6SPeter Krystad goto unlock; 2265cf7da0d6SPeter Krystad } 2266cf7da0d6SPeter Krystad 2267fa68018dSPaolo Abeni mptcp_token_destroy(msk); 2268fa68018dSPaolo Abeni inet_sk_state_store(sock->sk, TCP_LISTEN); 22695e20087dSFlorian Westphal sock_set_flag(sock->sk, SOCK_RCU_FREE); 22705e20087dSFlorian Westphal 2271cf7da0d6SPeter Krystad err = ssock->ops->listen(ssock, backlog); 2272cf7da0d6SPeter Krystad inet_sk_state_store(sock->sk, inet_sk_state_load(ssock->sk)); 2273cf7da0d6SPeter Krystad if (!err) 2274cf7da0d6SPeter Krystad mptcp_copy_inaddrs(sock->sk, ssock->sk); 2275cf7da0d6SPeter Krystad 2276cf7da0d6SPeter Krystad unlock: 2277cf7da0d6SPeter Krystad release_sock(sock->sk); 2278cf7da0d6SPeter Krystad return err; 2279cf7da0d6SPeter Krystad } 2280cf7da0d6SPeter Krystad 2281cf7da0d6SPeter Krystad static int mptcp_stream_accept(struct socket *sock, struct socket *newsock, 2282cf7da0d6SPeter Krystad int flags, bool kern) 2283cf7da0d6SPeter Krystad { 2284cf7da0d6SPeter Krystad struct mptcp_sock *msk = mptcp_sk(sock->sk); 2285cf7da0d6SPeter Krystad struct socket *ssock; 2286cf7da0d6SPeter Krystad int err; 2287cf7da0d6SPeter Krystad 2288cf7da0d6SPeter Krystad pr_debug("msk=%p", msk); 2289cf7da0d6SPeter Krystad 2290cf7da0d6SPeter Krystad lock_sock(sock->sk); 2291cf7da0d6SPeter Krystad if (sock->sk->sk_state != TCP_LISTEN) 2292cf7da0d6SPeter Krystad goto unlock_fail; 2293cf7da0d6SPeter Krystad 2294cf7da0d6SPeter Krystad ssock = __mptcp_nmpc_socket(msk); 2295cf7da0d6SPeter Krystad if (!ssock) 2296cf7da0d6SPeter Krystad goto unlock_fail; 2297cf7da0d6SPeter Krystad 22988a05661bSPaolo Abeni clear_bit(MPTCP_DATA_READY, &msk->flags); 2299cf7da0d6SPeter Krystad sock_hold(ssock->sk); 2300cf7da0d6SPeter Krystad release_sock(sock->sk); 2301cf7da0d6SPeter Krystad 2302cf7da0d6SPeter Krystad err = ssock->ops->accept(sock, newsock, flags, kern); 2303d2f77c53SPaolo Abeni if (err == 0 && !mptcp_is_tcpsk(newsock->sk)) { 2304cf7da0d6SPeter Krystad struct mptcp_sock *msk = mptcp_sk(newsock->sk); 2305cf7da0d6SPeter Krystad struct mptcp_subflow_context *subflow; 2306cf7da0d6SPeter Krystad 2307cf7da0d6SPeter Krystad /* set ssk->sk_socket of accept()ed flows to mptcp socket. 2308cf7da0d6SPeter Krystad * This is needed so NOSPACE flag can be set from tcp stack. 2309cf7da0d6SPeter Krystad */ 2310ec3edaa7SPeter Krystad __mptcp_flush_join_list(msk); 2311190f8b06SGeliang Tang mptcp_for_each_subflow(msk, subflow) { 2312cf7da0d6SPeter Krystad struct sock *ssk = mptcp_subflow_tcp_sock(subflow); 2313cf7da0d6SPeter Krystad 2314cf7da0d6SPeter Krystad if (!ssk->sk_socket) 2315cf7da0d6SPeter Krystad mptcp_sock_graft(ssk, newsock); 2316cf7da0d6SPeter Krystad } 2317cf7da0d6SPeter Krystad } 2318cf7da0d6SPeter Krystad 23198a05661bSPaolo Abeni if (inet_csk_listen_poll(ssock->sk)) 23208a05661bSPaolo Abeni set_bit(MPTCP_DATA_READY, &msk->flags); 2321cf7da0d6SPeter Krystad sock_put(ssock->sk); 2322cf7da0d6SPeter Krystad return err; 2323cf7da0d6SPeter Krystad 2324cf7da0d6SPeter Krystad unlock_fail: 2325cf7da0d6SPeter Krystad release_sock(sock->sk); 2326cf7da0d6SPeter Krystad return -EINVAL; 2327cf7da0d6SPeter Krystad } 2328cf7da0d6SPeter Krystad 23298a05661bSPaolo Abeni static __poll_t mptcp_check_readable(struct mptcp_sock *msk) 23308a05661bSPaolo Abeni { 23318a05661bSPaolo Abeni return test_bit(MPTCP_DATA_READY, &msk->flags) ? EPOLLIN | EPOLLRDNORM : 23328a05661bSPaolo Abeni 0; 23338a05661bSPaolo Abeni } 23348a05661bSPaolo Abeni 23352303f994SPeter Krystad static __poll_t mptcp_poll(struct file *file, struct socket *sock, 23362303f994SPeter Krystad struct poll_table_struct *wait) 23372303f994SPeter Krystad { 23381891c4a0SFlorian Westphal struct sock *sk = sock->sk; 23398ab183deSPaolo Abeni struct mptcp_sock *msk; 23402303f994SPeter Krystad __poll_t mask = 0; 23418a05661bSPaolo Abeni int state; 23422303f994SPeter Krystad 23431891c4a0SFlorian Westphal msk = mptcp_sk(sk); 23441891c4a0SFlorian Westphal sock_poll_wait(file, sock, wait); 23451891c4a0SFlorian Westphal 23468a05661bSPaolo Abeni state = inet_sk_state_load(sk); 23476719331cSPaolo Abeni pr_debug("msk=%p state=%d flags=%lx", msk, state, msk->flags); 23488a05661bSPaolo Abeni if (state == TCP_LISTEN) 23498a05661bSPaolo Abeni return mptcp_check_readable(msk); 23508a05661bSPaolo Abeni 23518a05661bSPaolo Abeni if (state != TCP_SYN_SENT && state != TCP_SYN_RECV) { 23528a05661bSPaolo Abeni mask |= mptcp_check_readable(msk); 235363561a40SPaolo Abeni if (test_bit(MPTCP_SEND_SPACE, &msk->flags)) 23541891c4a0SFlorian Westphal mask |= EPOLLOUT | EPOLLWRNORM; 23558a05661bSPaolo Abeni } 23561891c4a0SFlorian Westphal if (sk->sk_shutdown & RCV_SHUTDOWN) 23571891c4a0SFlorian Westphal mask |= EPOLLIN | EPOLLRDNORM | EPOLLRDHUP; 23581891c4a0SFlorian Westphal 23592303f994SPeter Krystad return mask; 23602303f994SPeter Krystad } 23612303f994SPeter Krystad 236221498490SPeter Krystad static int mptcp_shutdown(struct socket *sock, int how) 236321498490SPeter Krystad { 236421498490SPeter Krystad struct mptcp_sock *msk = mptcp_sk(sock->sk); 236521498490SPeter Krystad struct mptcp_subflow_context *subflow; 236621498490SPeter Krystad int ret = 0; 236721498490SPeter Krystad 236821498490SPeter Krystad pr_debug("sk=%p, how=%d", msk, how); 236921498490SPeter Krystad 237021498490SPeter Krystad lock_sock(sock->sk); 237121498490SPeter Krystad 237221498490SPeter Krystad how++; 237321498490SPeter Krystad if ((how & ~SHUTDOWN_MASK) || !how) { 237421498490SPeter Krystad ret = -EINVAL; 237521498490SPeter Krystad goto out_unlock; 237621498490SPeter Krystad } 237721498490SPeter Krystad 237821498490SPeter Krystad if (sock->state == SS_CONNECTING) { 237921498490SPeter Krystad if ((1 << sock->sk->sk_state) & 238021498490SPeter Krystad (TCPF_SYN_SENT | TCPF_SYN_RECV | TCPF_CLOSE)) 238121498490SPeter Krystad sock->state = SS_DISCONNECTING; 238221498490SPeter Krystad else 238321498490SPeter Krystad sock->state = SS_CONNECTED; 238421498490SPeter Krystad } 238521498490SPeter Krystad 238643b54c6eSMat Martineau /* If we've already sent a FIN, or it's a closed state, skip this. */ 238743b54c6eSMat Martineau if (__mptcp_check_fallback(msk)) { 238843b54c6eSMat Martineau if (how == SHUT_WR || how == SHUT_RDWR) 238943b54c6eSMat Martineau inet_sk_state_store(sock->sk, TCP_FIN_WAIT1); 23907279da61SMat Martineau 239121498490SPeter Krystad mptcp_for_each_subflow(msk, subflow) { 239221498490SPeter Krystad struct sock *tcp_sk = mptcp_subflow_tcp_sock(subflow); 239321498490SPeter Krystad 239443b54c6eSMat Martineau mptcp_subflow_shutdown(sock->sk, tcp_sk, how); 239543b54c6eSMat Martineau } 239643b54c6eSMat Martineau } else if ((how & SEND_SHUTDOWN) && 239743b54c6eSMat Martineau ((1 << sock->sk->sk_state) & 239843b54c6eSMat Martineau (TCPF_ESTABLISHED | TCPF_SYN_SENT | 239943b54c6eSMat Martineau TCPF_SYN_RECV | TCPF_CLOSE_WAIT)) && 240043b54c6eSMat Martineau mptcp_close_state(sock->sk)) { 240143b54c6eSMat Martineau __mptcp_flush_join_list(msk); 240243b54c6eSMat Martineau 240343b54c6eSMat Martineau WRITE_ONCE(msk->write_seq, msk->write_seq + 1); 240443b54c6eSMat Martineau WRITE_ONCE(msk->snd_data_fin_enable, 1); 240543b54c6eSMat Martineau 240643b54c6eSMat Martineau mptcp_for_each_subflow(msk, subflow) { 240743b54c6eSMat Martineau struct sock *tcp_sk = mptcp_subflow_tcp_sock(subflow); 240843b54c6eSMat Martineau 240943b54c6eSMat Martineau mptcp_subflow_shutdown(sock->sk, tcp_sk, how); 241043b54c6eSMat Martineau } 241121498490SPeter Krystad } 241221498490SPeter Krystad 2413e1ff9e82SDavide Caratti /* Wake up anyone sleeping in poll. */ 2414e1ff9e82SDavide Caratti sock->sk->sk_state_change(sock->sk); 2415e1ff9e82SDavide Caratti 241621498490SPeter Krystad out_unlock: 241721498490SPeter Krystad release_sock(sock->sk); 241821498490SPeter Krystad 241921498490SPeter Krystad return ret; 242021498490SPeter Krystad } 242121498490SPeter Krystad 2422e42f1ac6SFlorian Westphal static const struct proto_ops mptcp_stream_ops = { 2423e42f1ac6SFlorian Westphal .family = PF_INET, 2424e42f1ac6SFlorian Westphal .owner = THIS_MODULE, 2425e42f1ac6SFlorian Westphal .release = inet_release, 2426e42f1ac6SFlorian Westphal .bind = mptcp_bind, 2427e42f1ac6SFlorian Westphal .connect = mptcp_stream_connect, 2428e42f1ac6SFlorian Westphal .socketpair = sock_no_socketpair, 2429e42f1ac6SFlorian Westphal .accept = mptcp_stream_accept, 2430d2f77c53SPaolo Abeni .getname = inet_getname, 2431e42f1ac6SFlorian Westphal .poll = mptcp_poll, 2432e42f1ac6SFlorian Westphal .ioctl = inet_ioctl, 2433e42f1ac6SFlorian Westphal .gettstamp = sock_gettstamp, 2434e42f1ac6SFlorian Westphal .listen = mptcp_listen, 2435e42f1ac6SFlorian Westphal .shutdown = mptcp_shutdown, 2436e42f1ac6SFlorian Westphal .setsockopt = sock_common_setsockopt, 2437e42f1ac6SFlorian Westphal .getsockopt = sock_common_getsockopt, 2438e42f1ac6SFlorian Westphal .sendmsg = inet_sendmsg, 2439e42f1ac6SFlorian Westphal .recvmsg = inet_recvmsg, 2440e42f1ac6SFlorian Westphal .mmap = sock_no_mmap, 2441e42f1ac6SFlorian Westphal .sendpage = inet_sendpage, 2442e42f1ac6SFlorian Westphal }; 24432303f994SPeter Krystad 2444f870fa0bSMat Martineau static struct inet_protosw mptcp_protosw = { 2445f870fa0bSMat Martineau .type = SOCK_STREAM, 2446f870fa0bSMat Martineau .protocol = IPPROTO_MPTCP, 2447f870fa0bSMat Martineau .prot = &mptcp_prot, 24482303f994SPeter Krystad .ops = &mptcp_stream_ops, 24492303f994SPeter Krystad .flags = INET_PROTOSW_ICSK, 2450f870fa0bSMat Martineau }; 2451f870fa0bSMat Martineau 2452d39dcecaSPaolo Abeni void __init mptcp_proto_init(void) 2453f870fa0bSMat Martineau { 24542303f994SPeter Krystad mptcp_prot.h.hashinfo = tcp_prot.h.hashinfo; 24552303f994SPeter Krystad 2456d027236cSPaolo Abeni if (percpu_counter_init(&mptcp_sockets_allocated, 0, GFP_KERNEL)) 2457d027236cSPaolo Abeni panic("Failed to allocate MPTCP pcpu counter\n"); 2458d027236cSPaolo Abeni 24592303f994SPeter Krystad mptcp_subflow_init(); 24601b1c7a0eSPeter Krystad mptcp_pm_init(); 24612c5ebd00SPaolo Abeni mptcp_token_init(); 24622303f994SPeter Krystad 2463f870fa0bSMat Martineau if (proto_register(&mptcp_prot, 1) != 0) 2464f870fa0bSMat Martineau panic("Failed to register MPTCP proto.\n"); 2465f870fa0bSMat Martineau 2466f870fa0bSMat Martineau inet_register_protosw(&mptcp_protosw); 24676771bfd9SFlorian Westphal 24686771bfd9SFlorian Westphal BUILD_BUG_ON(sizeof(struct mptcp_skb_cb) > sizeof_field(struct sk_buff, cb)); 2469f870fa0bSMat Martineau } 2470f870fa0bSMat Martineau 2471f870fa0bSMat Martineau #if IS_ENABLED(CONFIG_MPTCP_IPV6) 2472e42f1ac6SFlorian Westphal static const struct proto_ops mptcp_v6_stream_ops = { 2473e42f1ac6SFlorian Westphal .family = PF_INET6, 2474e42f1ac6SFlorian Westphal .owner = THIS_MODULE, 2475e42f1ac6SFlorian Westphal .release = inet6_release, 2476e42f1ac6SFlorian Westphal .bind = mptcp_bind, 2477e42f1ac6SFlorian Westphal .connect = mptcp_stream_connect, 2478e42f1ac6SFlorian Westphal .socketpair = sock_no_socketpair, 2479e42f1ac6SFlorian Westphal .accept = mptcp_stream_accept, 2480d2f77c53SPaolo Abeni .getname = inet6_getname, 2481e42f1ac6SFlorian Westphal .poll = mptcp_poll, 2482e42f1ac6SFlorian Westphal .ioctl = inet6_ioctl, 2483e42f1ac6SFlorian Westphal .gettstamp = sock_gettstamp, 2484e42f1ac6SFlorian Westphal .listen = mptcp_listen, 2485e42f1ac6SFlorian Westphal .shutdown = mptcp_shutdown, 2486e42f1ac6SFlorian Westphal .setsockopt = sock_common_setsockopt, 2487e42f1ac6SFlorian Westphal .getsockopt = sock_common_getsockopt, 2488e42f1ac6SFlorian Westphal .sendmsg = inet6_sendmsg, 2489e42f1ac6SFlorian Westphal .recvmsg = inet6_recvmsg, 2490e42f1ac6SFlorian Westphal .mmap = sock_no_mmap, 2491e42f1ac6SFlorian Westphal .sendpage = inet_sendpage, 2492e42f1ac6SFlorian Westphal #ifdef CONFIG_COMPAT 24933986912fSChristoph Hellwig .compat_ioctl = inet6_compat_ioctl, 2494e42f1ac6SFlorian Westphal #endif 2495e42f1ac6SFlorian Westphal }; 2496e42f1ac6SFlorian Westphal 2497f870fa0bSMat Martineau static struct proto mptcp_v6_prot; 2498f870fa0bSMat Martineau 249979c0949eSPeter Krystad static void mptcp_v6_destroy(struct sock *sk) 250079c0949eSPeter Krystad { 250179c0949eSPeter Krystad mptcp_destroy(sk); 250279c0949eSPeter Krystad inet6_destroy_sock(sk); 250379c0949eSPeter Krystad } 250479c0949eSPeter Krystad 2505f870fa0bSMat Martineau static struct inet_protosw mptcp_v6_protosw = { 2506f870fa0bSMat Martineau .type = SOCK_STREAM, 2507f870fa0bSMat Martineau .protocol = IPPROTO_MPTCP, 2508f870fa0bSMat Martineau .prot = &mptcp_v6_prot, 25092303f994SPeter Krystad .ops = &mptcp_v6_stream_ops, 2510f870fa0bSMat Martineau .flags = INET_PROTOSW_ICSK, 2511f870fa0bSMat Martineau }; 2512f870fa0bSMat Martineau 2513d39dcecaSPaolo Abeni int __init mptcp_proto_v6_init(void) 2514f870fa0bSMat Martineau { 2515f870fa0bSMat Martineau int err; 2516f870fa0bSMat Martineau 2517f870fa0bSMat Martineau mptcp_v6_prot = mptcp_prot; 2518f870fa0bSMat Martineau strcpy(mptcp_v6_prot.name, "MPTCPv6"); 2519f870fa0bSMat Martineau mptcp_v6_prot.slab = NULL; 252079c0949eSPeter Krystad mptcp_v6_prot.destroy = mptcp_v6_destroy; 2521b0519de8SFlorian Westphal mptcp_v6_prot.obj_size = sizeof(struct mptcp6_sock); 2522f870fa0bSMat Martineau 2523f870fa0bSMat Martineau err = proto_register(&mptcp_v6_prot, 1); 2524f870fa0bSMat Martineau if (err) 2525f870fa0bSMat Martineau return err; 2526f870fa0bSMat Martineau 2527f870fa0bSMat Martineau err = inet6_register_protosw(&mptcp_v6_protosw); 2528f870fa0bSMat Martineau if (err) 2529f870fa0bSMat Martineau proto_unregister(&mptcp_v6_prot); 2530f870fa0bSMat Martineau 2531f870fa0bSMat Martineau return err; 2532f870fa0bSMat Martineau } 2533f870fa0bSMat Martineau #endif 2534