1f870fa0bSMat Martineau // SPDX-License-Identifier: GPL-2.0 2f870fa0bSMat Martineau /* Multipath TCP 3f870fa0bSMat Martineau * 4f870fa0bSMat Martineau * Copyright (c) 2017 - 2019, Intel Corporation. 5f870fa0bSMat Martineau */ 6f870fa0bSMat Martineau 7f870fa0bSMat Martineau #define pr_fmt(fmt) "MPTCP: " fmt 8f870fa0bSMat Martineau 9f870fa0bSMat Martineau #include <linux/kernel.h> 10f870fa0bSMat Martineau #include <linux/module.h> 11f870fa0bSMat Martineau #include <linux/netdevice.h> 127a6a6cbcSPaolo Abeni #include <linux/sched/signal.h> 137a6a6cbcSPaolo Abeni #include <linux/atomic.h> 14ad98dd37SFlorian Westphal #include <linux/igmp.h> 15f870fa0bSMat Martineau #include <net/sock.h> 16f870fa0bSMat Martineau #include <net/inet_common.h> 17f870fa0bSMat Martineau #include <net/inet_hashtables.h> 18f870fa0bSMat Martineau #include <net/protocol.h> 19f870fa0bSMat Martineau #include <net/tcp.h> 203721b9b6SMat Martineau #include <net/tcp_states.h> 21cf7da0d6SPeter Krystad #if IS_ENABLED(CONFIG_MPTCP_IPV6) 22cf7da0d6SPeter Krystad #include <net/transp_v6.h> 23ad98dd37SFlorian Westphal #include <net/addrconf.h> 24cf7da0d6SPeter Krystad #endif 25f870fa0bSMat Martineau #include <net/mptcp.h> 26e16163b6SPaolo Abeni #include <net/xfrm.h> 27f870fa0bSMat Martineau #include "protocol.h" 28fc518953SFlorian Westphal #include "mib.h" 29f870fa0bSMat Martineau 30b0519de8SFlorian Westphal #if IS_ENABLED(CONFIG_MPTCP_IPV6) 31b0519de8SFlorian Westphal struct mptcp6_sock { 32b0519de8SFlorian Westphal struct mptcp_sock msk; 33b0519de8SFlorian Westphal struct ipv6_pinfo np; 34b0519de8SFlorian Westphal }; 35b0519de8SFlorian Westphal #endif 36b0519de8SFlorian Westphal 376771bfd9SFlorian Westphal struct mptcp_skb_cb { 38ab174ad8SPaolo Abeni u64 map_seq; 39ab174ad8SPaolo Abeni u64 end_seq; 406771bfd9SFlorian Westphal u32 offset; 416771bfd9SFlorian Westphal }; 426771bfd9SFlorian Westphal 436771bfd9SFlorian Westphal #define MPTCP_SKB_CB(__skb) ((struct mptcp_skb_cb *)&((__skb)->cb[0])) 446771bfd9SFlorian Westphal 45d027236cSPaolo Abeni static struct percpu_counter mptcp_sockets_allocated; 46d027236cSPaolo Abeni 47e16163b6SPaolo Abeni static void __mptcp_destroy_sock(struct sock *sk); 48d9ca1de8SPaolo Abeni static void __mptcp_check_send_data_fin(struct sock *sk); 49e16163b6SPaolo Abeni 50b19bc294SPaolo Abeni DEFINE_PER_CPU(struct mptcp_delegated_action, mptcp_delegated_actions); 51b19bc294SPaolo Abeni static struct net_device mptcp_napi_dev; 52b19bc294SPaolo Abeni 532303f994SPeter Krystad /* If msk has an initial subflow socket, and the MP_CAPABLE handshake has not 542303f994SPeter Krystad * completed yet or has failed, return the subflow socket. 552303f994SPeter Krystad * Otherwise return NULL. 562303f994SPeter Krystad */ 571729cf18SGeliang Tang struct socket *__mptcp_nmpc_socket(const struct mptcp_sock *msk) 582303f994SPeter Krystad { 59d22f4988SChristoph Paasch if (!msk->subflow || READ_ONCE(msk->can_ack)) 602303f994SPeter Krystad return NULL; 612303f994SPeter Krystad 622303f994SPeter Krystad return msk->subflow; 632303f994SPeter Krystad } 642303f994SPeter Krystad 656f8a612aSFlorian Westphal /* Returns end sequence number of the receiver's advertised window */ 666f8a612aSFlorian Westphal static u64 mptcp_wnd_end(const struct mptcp_sock *msk) 676f8a612aSFlorian Westphal { 687439d687SPaolo Abeni return READ_ONCE(msk->wnd_end); 696f8a612aSFlorian Westphal } 706f8a612aSFlorian Westphal 71d2f77c53SPaolo Abeni static bool mptcp_is_tcpsk(struct sock *sk) 720b4f33deSFlorian Westphal { 730b4f33deSFlorian Westphal struct socket *sock = sk->sk_socket; 740b4f33deSFlorian Westphal 750b4f33deSFlorian Westphal if (unlikely(sk->sk_prot == &tcp_prot)) { 760b4f33deSFlorian Westphal /* we are being invoked after mptcp_accept() has 770b4f33deSFlorian Westphal * accepted a non-mp-capable flow: sk is a tcp_sk, 780b4f33deSFlorian Westphal * not an mptcp one. 790b4f33deSFlorian Westphal * 800b4f33deSFlorian Westphal * Hand the socket over to tcp so all further socket ops 810b4f33deSFlorian Westphal * bypass mptcp. 820b4f33deSFlorian Westphal */ 830b4f33deSFlorian Westphal sock->ops = &inet_stream_ops; 84d2f77c53SPaolo Abeni return true; 850b4f33deSFlorian Westphal #if IS_ENABLED(CONFIG_MPTCP_IPV6) 860b4f33deSFlorian Westphal } else if (unlikely(sk->sk_prot == &tcpv6_prot)) { 870b4f33deSFlorian Westphal sock->ops = &inet6_stream_ops; 88d2f77c53SPaolo Abeni return true; 890b4f33deSFlorian Westphal #endif 900b4f33deSFlorian Westphal } 910b4f33deSFlorian Westphal 92d2f77c53SPaolo Abeni return false; 930b4f33deSFlorian Westphal } 940b4f33deSFlorian Westphal 9576660afbSPaolo Abeni static struct sock *__mptcp_tcp_fallback(struct mptcp_sock *msk) 96cec37a6eSPeter Krystad { 97cec37a6eSPeter Krystad sock_owned_by_me((const struct sock *)msk); 98cec37a6eSPeter Krystad 99e1ff9e82SDavide Caratti if (likely(!__mptcp_check_fallback(msk))) 100cec37a6eSPeter Krystad return NULL; 101cec37a6eSPeter Krystad 10276660afbSPaolo Abeni return msk->first; 1038ab183deSPaolo Abeni } 1048ab183deSPaolo Abeni 105fa68018dSPaolo Abeni static int __mptcp_socket_create(struct mptcp_sock *msk) 1062303f994SPeter Krystad { 1072303f994SPeter Krystad struct mptcp_subflow_context *subflow; 1082303f994SPeter Krystad struct sock *sk = (struct sock *)msk; 1092303f994SPeter Krystad struct socket *ssock; 1102303f994SPeter Krystad int err; 1112303f994SPeter Krystad 1122303f994SPeter Krystad err = mptcp_subflow_create_socket(sk, &ssock); 1132303f994SPeter Krystad if (err) 114fa68018dSPaolo Abeni return err; 1152303f994SPeter Krystad 1168ab183deSPaolo Abeni msk->first = ssock->sk; 1172303f994SPeter Krystad msk->subflow = ssock; 1182303f994SPeter Krystad subflow = mptcp_subflow_ctx(ssock->sk); 119cec37a6eSPeter Krystad list_add(&subflow->node, &msk->conn_list); 120e16163b6SPaolo Abeni sock_hold(ssock->sk); 1212303f994SPeter Krystad subflow->request_mptcp = 1; 122866f26f2SPaolo Abeni mptcp_sock_graft(msk->first, sk->sk_socket); 123e1ff9e82SDavide Caratti 124fa68018dSPaolo Abeni return 0; 1252303f994SPeter Krystad } 1262303f994SPeter Krystad 127ab174ad8SPaolo Abeni static void mptcp_drop(struct sock *sk, struct sk_buff *skb) 128ab174ad8SPaolo Abeni { 129ab174ad8SPaolo Abeni sk_drops_add(sk, skb); 130ab174ad8SPaolo Abeni __kfree_skb(skb); 131ab174ad8SPaolo Abeni } 132ab174ad8SPaolo Abeni 1338268ed4cSPaolo Abeni static bool mptcp_try_coalesce(struct sock *sk, struct sk_buff *to, 1348268ed4cSPaolo Abeni struct sk_buff *from) 1358268ed4cSPaolo Abeni { 1368268ed4cSPaolo Abeni bool fragstolen; 1378268ed4cSPaolo Abeni int delta; 1388268ed4cSPaolo Abeni 1398268ed4cSPaolo Abeni if (MPTCP_SKB_CB(from)->offset || 1408268ed4cSPaolo Abeni !skb_try_coalesce(to, from, &fragstolen, &delta)) 1418268ed4cSPaolo Abeni return false; 1428268ed4cSPaolo Abeni 14306242e44SPaolo Abeni pr_debug("colesced seq %llx into %llx new len %d new end seq %llx", 14406242e44SPaolo Abeni MPTCP_SKB_CB(from)->map_seq, MPTCP_SKB_CB(to)->map_seq, 14506242e44SPaolo Abeni to->len, MPTCP_SKB_CB(from)->end_seq); 146ab174ad8SPaolo Abeni MPTCP_SKB_CB(to)->end_seq = MPTCP_SKB_CB(from)->end_seq; 1478268ed4cSPaolo Abeni kfree_skb_partial(from, fragstolen); 1488268ed4cSPaolo Abeni atomic_add(delta, &sk->sk_rmem_alloc); 1498268ed4cSPaolo Abeni sk_mem_charge(sk, delta); 1508268ed4cSPaolo Abeni return true; 1518268ed4cSPaolo Abeni } 1528268ed4cSPaolo Abeni 153ab174ad8SPaolo Abeni static bool mptcp_ooo_try_coalesce(struct mptcp_sock *msk, struct sk_buff *to, 154ab174ad8SPaolo Abeni struct sk_buff *from) 1556771bfd9SFlorian Westphal { 156ab174ad8SPaolo Abeni if (MPTCP_SKB_CB(from)->map_seq != MPTCP_SKB_CB(to)->end_seq) 157ab174ad8SPaolo Abeni return false; 158ab174ad8SPaolo Abeni 159ab174ad8SPaolo Abeni return mptcp_try_coalesce((struct sock *)msk, to, from); 160ab174ad8SPaolo Abeni } 161ab174ad8SPaolo Abeni 162ab174ad8SPaolo Abeni /* "inspired" by tcp_data_queue_ofo(), main differences: 163ab174ad8SPaolo Abeni * - use mptcp seqs 164ab174ad8SPaolo Abeni * - don't cope with sacks 165ab174ad8SPaolo Abeni */ 166ab174ad8SPaolo Abeni static void mptcp_data_queue_ofo(struct mptcp_sock *msk, struct sk_buff *skb) 167ab174ad8SPaolo Abeni { 168ab174ad8SPaolo Abeni struct sock *sk = (struct sock *)msk; 169ab174ad8SPaolo Abeni struct rb_node **p, *parent; 170ab174ad8SPaolo Abeni u64 seq, end_seq, max_seq; 171ab174ad8SPaolo Abeni struct sk_buff *skb1; 172ab174ad8SPaolo Abeni 173ab174ad8SPaolo Abeni seq = MPTCP_SKB_CB(skb)->map_seq; 174ab174ad8SPaolo Abeni end_seq = MPTCP_SKB_CB(skb)->end_seq; 175fa3fe2b1SFlorian Westphal max_seq = READ_ONCE(msk->rcv_wnd_sent); 176ab174ad8SPaolo Abeni 17706242e44SPaolo Abeni pr_debug("msk=%p seq=%llx limit=%llx empty=%d", msk, seq, max_seq, 17806242e44SPaolo Abeni RB_EMPTY_ROOT(&msk->out_of_order_queue)); 179fa3fe2b1SFlorian Westphal if (after64(end_seq, max_seq)) { 180ab174ad8SPaolo Abeni /* out of window */ 181ab174ad8SPaolo Abeni mptcp_drop(sk, skb); 182fa3fe2b1SFlorian Westphal pr_debug("oow by %lld, rcv_wnd_sent %llu\n", 183fa3fe2b1SFlorian Westphal (unsigned long long)end_seq - (unsigned long)max_seq, 184fa3fe2b1SFlorian Westphal (unsigned long long)msk->rcv_wnd_sent); 18506242e44SPaolo Abeni MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_NODSSWINDOW); 186ab174ad8SPaolo Abeni return; 187ab174ad8SPaolo Abeni } 188ab174ad8SPaolo Abeni 189ab174ad8SPaolo Abeni p = &msk->out_of_order_queue.rb_node; 19006242e44SPaolo Abeni MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_OFOQUEUE); 191ab174ad8SPaolo Abeni if (RB_EMPTY_ROOT(&msk->out_of_order_queue)) { 192ab174ad8SPaolo Abeni rb_link_node(&skb->rbnode, NULL, p); 193ab174ad8SPaolo Abeni rb_insert_color(&skb->rbnode, &msk->out_of_order_queue); 194ab174ad8SPaolo Abeni msk->ooo_last_skb = skb; 195ab174ad8SPaolo Abeni goto end; 196ab174ad8SPaolo Abeni } 197ab174ad8SPaolo Abeni 198ab174ad8SPaolo Abeni /* with 2 subflows, adding at end of ooo queue is quite likely 199ab174ad8SPaolo Abeni * Use of ooo_last_skb avoids the O(Log(N)) rbtree lookup. 200ab174ad8SPaolo Abeni */ 20106242e44SPaolo Abeni if (mptcp_ooo_try_coalesce(msk, msk->ooo_last_skb, skb)) { 20206242e44SPaolo Abeni MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_OFOMERGE); 20306242e44SPaolo Abeni MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_OFOQUEUETAIL); 204ab174ad8SPaolo Abeni return; 20506242e44SPaolo Abeni } 206ab174ad8SPaolo Abeni 207ab174ad8SPaolo Abeni /* Can avoid an rbtree lookup if we are adding skb after ooo_last_skb */ 208ab174ad8SPaolo Abeni if (!before64(seq, MPTCP_SKB_CB(msk->ooo_last_skb)->end_seq)) { 20906242e44SPaolo Abeni MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_OFOQUEUETAIL); 210ab174ad8SPaolo Abeni parent = &msk->ooo_last_skb->rbnode; 211ab174ad8SPaolo Abeni p = &parent->rb_right; 212ab174ad8SPaolo Abeni goto insert; 213ab174ad8SPaolo Abeni } 214ab174ad8SPaolo Abeni 215ab174ad8SPaolo Abeni /* Find place to insert this segment. Handle overlaps on the way. */ 216ab174ad8SPaolo Abeni parent = NULL; 217ab174ad8SPaolo Abeni while (*p) { 218ab174ad8SPaolo Abeni parent = *p; 219ab174ad8SPaolo Abeni skb1 = rb_to_skb(parent); 220ab174ad8SPaolo Abeni if (before64(seq, MPTCP_SKB_CB(skb1)->map_seq)) { 221ab174ad8SPaolo Abeni p = &parent->rb_left; 222ab174ad8SPaolo Abeni continue; 223ab174ad8SPaolo Abeni } 224ab174ad8SPaolo Abeni if (before64(seq, MPTCP_SKB_CB(skb1)->end_seq)) { 225ab174ad8SPaolo Abeni if (!after64(end_seq, MPTCP_SKB_CB(skb1)->end_seq)) { 226ab174ad8SPaolo Abeni /* All the bits are present. Drop. */ 227ab174ad8SPaolo Abeni mptcp_drop(sk, skb); 22806242e44SPaolo Abeni MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_DUPDATA); 229ab174ad8SPaolo Abeni return; 230ab174ad8SPaolo Abeni } 231ab174ad8SPaolo Abeni if (after64(seq, MPTCP_SKB_CB(skb1)->map_seq)) { 232ab174ad8SPaolo Abeni /* partial overlap: 233ab174ad8SPaolo Abeni * | skb | 234ab174ad8SPaolo Abeni * | skb1 | 235ab174ad8SPaolo Abeni * continue traversing 236ab174ad8SPaolo Abeni */ 237ab174ad8SPaolo Abeni } else { 238ab174ad8SPaolo Abeni /* skb's seq == skb1's seq and skb covers skb1. 239ab174ad8SPaolo Abeni * Replace skb1 with skb. 240ab174ad8SPaolo Abeni */ 241ab174ad8SPaolo Abeni rb_replace_node(&skb1->rbnode, &skb->rbnode, 242ab174ad8SPaolo Abeni &msk->out_of_order_queue); 243ab174ad8SPaolo Abeni mptcp_drop(sk, skb1); 24406242e44SPaolo Abeni MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_DUPDATA); 245ab174ad8SPaolo Abeni goto merge_right; 246ab174ad8SPaolo Abeni } 247ab174ad8SPaolo Abeni } else if (mptcp_ooo_try_coalesce(msk, skb1, skb)) { 24806242e44SPaolo Abeni MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_OFOMERGE); 249ab174ad8SPaolo Abeni return; 250ab174ad8SPaolo Abeni } 251ab174ad8SPaolo Abeni p = &parent->rb_right; 252ab174ad8SPaolo Abeni } 25306242e44SPaolo Abeni 254ab174ad8SPaolo Abeni insert: 255ab174ad8SPaolo Abeni /* Insert segment into RB tree. */ 256ab174ad8SPaolo Abeni rb_link_node(&skb->rbnode, parent, p); 257ab174ad8SPaolo Abeni rb_insert_color(&skb->rbnode, &msk->out_of_order_queue); 258ab174ad8SPaolo Abeni 259ab174ad8SPaolo Abeni merge_right: 260ab174ad8SPaolo Abeni /* Remove other segments covered by skb. */ 261ab174ad8SPaolo Abeni while ((skb1 = skb_rb_next(skb)) != NULL) { 262ab174ad8SPaolo Abeni if (before64(end_seq, MPTCP_SKB_CB(skb1)->end_seq)) 263ab174ad8SPaolo Abeni break; 264ab174ad8SPaolo Abeni rb_erase(&skb1->rbnode, &msk->out_of_order_queue); 265ab174ad8SPaolo Abeni mptcp_drop(sk, skb1); 26606242e44SPaolo Abeni MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_DUPDATA); 267ab174ad8SPaolo Abeni } 268ab174ad8SPaolo Abeni /* If there is no skb after us, we are the last_skb ! */ 269ab174ad8SPaolo Abeni if (!skb1) 270ab174ad8SPaolo Abeni msk->ooo_last_skb = skb; 271ab174ad8SPaolo Abeni 272ab174ad8SPaolo Abeni end: 273ab174ad8SPaolo Abeni skb_condense(skb); 274ab174ad8SPaolo Abeni skb_set_owner_r(skb, sk); 275ab174ad8SPaolo Abeni } 276ab174ad8SPaolo Abeni 277ab174ad8SPaolo Abeni static bool __mptcp_move_skb(struct mptcp_sock *msk, struct sock *ssk, 278ab174ad8SPaolo Abeni struct sk_buff *skb, unsigned int offset, 279ab174ad8SPaolo Abeni size_t copy_len) 280ab174ad8SPaolo Abeni { 281ab174ad8SPaolo Abeni struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk); 2826771bfd9SFlorian Westphal struct sock *sk = (struct sock *)msk; 2834e637c70SFlorian Westphal struct sk_buff *tail; 2846771bfd9SFlorian Westphal 2856771bfd9SFlorian Westphal __skb_unlink(skb, &ssk->sk_receive_queue); 2864e637c70SFlorian Westphal 2874e637c70SFlorian Westphal skb_ext_reset(skb); 2884e637c70SFlorian Westphal skb_orphan(skb); 2894e637c70SFlorian Westphal 2909c3f94e1SPaolo Abeni /* try to fetch required memory from subflow */ 2919c3f94e1SPaolo Abeni if (!sk_rmem_schedule(sk, skb, skb->truesize)) { 2929c3f94e1SPaolo Abeni if (ssk->sk_forward_alloc < skb->truesize) 2939c3f94e1SPaolo Abeni goto drop; 2949c3f94e1SPaolo Abeni __sk_mem_reclaim(ssk, skb->truesize); 2959c3f94e1SPaolo Abeni if (!sk_rmem_schedule(sk, skb, skb->truesize)) 2969c3f94e1SPaolo Abeni goto drop; 2979c3f94e1SPaolo Abeni } 2989c3f94e1SPaolo Abeni 299ab174ad8SPaolo Abeni /* the skb map_seq accounts for the skb offset: 300ab174ad8SPaolo Abeni * mptcp_subflow_get_mapped_dsn() is based on the current tp->copied_seq 301ab174ad8SPaolo Abeni * value 302ab174ad8SPaolo Abeni */ 303ab174ad8SPaolo Abeni MPTCP_SKB_CB(skb)->map_seq = mptcp_subflow_get_mapped_dsn(subflow); 304ab174ad8SPaolo Abeni MPTCP_SKB_CB(skb)->end_seq = MPTCP_SKB_CB(skb)->map_seq + copy_len; 305ab174ad8SPaolo Abeni MPTCP_SKB_CB(skb)->offset = offset; 306ab174ad8SPaolo Abeni 307ab174ad8SPaolo Abeni if (MPTCP_SKB_CB(skb)->map_seq == msk->ack_seq) { 308ab174ad8SPaolo Abeni /* in sequence */ 309917944daSMat Martineau WRITE_ONCE(msk->ack_seq, msk->ack_seq + copy_len); 3104e637c70SFlorian Westphal tail = skb_peek_tail(&sk->sk_receive_queue); 3118268ed4cSPaolo Abeni if (tail && mptcp_try_coalesce(sk, tail, skb)) 312ab174ad8SPaolo Abeni return true; 3134e637c70SFlorian Westphal 314600911ffSFlorian Westphal skb_set_owner_r(skb, sk); 3156771bfd9SFlorian Westphal __skb_queue_tail(&sk->sk_receive_queue, skb); 316ab174ad8SPaolo Abeni return true; 317ab174ad8SPaolo Abeni } else if (after64(MPTCP_SKB_CB(skb)->map_seq, msk->ack_seq)) { 318ab174ad8SPaolo Abeni mptcp_data_queue_ofo(msk, skb); 319ab174ad8SPaolo Abeni return false; 320ab174ad8SPaolo Abeni } 321ab174ad8SPaolo Abeni 322ab174ad8SPaolo Abeni /* old data, keep it simple and drop the whole pkt, sender 323ab174ad8SPaolo Abeni * will retransmit as needed, if needed. 324ab174ad8SPaolo Abeni */ 32506242e44SPaolo Abeni MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_DUPDATA); 3269c3f94e1SPaolo Abeni drop: 327ab174ad8SPaolo Abeni mptcp_drop(sk, skb); 328ab174ad8SPaolo Abeni return false; 3296771bfd9SFlorian Westphal } 3306771bfd9SFlorian Westphal 33116a9a9daSMat Martineau static void mptcp_stop_timer(struct sock *sk) 33216a9a9daSMat Martineau { 33316a9a9daSMat Martineau struct inet_connection_sock *icsk = inet_csk(sk); 33416a9a9daSMat Martineau 33516a9a9daSMat Martineau sk_stop_timer(sk, &icsk->icsk_retransmit_timer); 33616a9a9daSMat Martineau mptcp_sk(sk)->timer_ival = 0; 33716a9a9daSMat Martineau } 33816a9a9daSMat Martineau 339e16163b6SPaolo Abeni static void mptcp_close_wake_up(struct sock *sk) 340e16163b6SPaolo Abeni { 341e16163b6SPaolo Abeni if (sock_flag(sk, SOCK_DEAD)) 342e16163b6SPaolo Abeni return; 343e16163b6SPaolo Abeni 344e16163b6SPaolo Abeni sk->sk_state_change(sk); 345e16163b6SPaolo Abeni if (sk->sk_shutdown == SHUTDOWN_MASK || 346e16163b6SPaolo Abeni sk->sk_state == TCP_CLOSE) 347e16163b6SPaolo Abeni sk_wake_async(sk, SOCK_WAKE_WAITD, POLL_HUP); 348e16163b6SPaolo Abeni else 349e16163b6SPaolo Abeni sk_wake_async(sk, SOCK_WAKE_WAITD, POLL_IN); 350e16163b6SPaolo Abeni } 351e16163b6SPaolo Abeni 3526e628cd3SPaolo Abeni static bool mptcp_pending_data_fin_ack(struct sock *sk) 3536e628cd3SPaolo Abeni { 3546e628cd3SPaolo Abeni struct mptcp_sock *msk = mptcp_sk(sk); 3556e628cd3SPaolo Abeni 3566e628cd3SPaolo Abeni return !__mptcp_check_fallback(msk) && 3576e628cd3SPaolo Abeni ((1 << sk->sk_state) & 3586e628cd3SPaolo Abeni (TCPF_FIN_WAIT1 | TCPF_CLOSING | TCPF_LAST_ACK)) && 3596e628cd3SPaolo Abeni msk->write_seq == READ_ONCE(msk->snd_una); 3606e628cd3SPaolo Abeni } 3616e628cd3SPaolo Abeni 36216a9a9daSMat Martineau static void mptcp_check_data_fin_ack(struct sock *sk) 36316a9a9daSMat Martineau { 36416a9a9daSMat Martineau struct mptcp_sock *msk = mptcp_sk(sk); 36516a9a9daSMat Martineau 36616a9a9daSMat Martineau /* Look for an acknowledged DATA_FIN */ 3676e628cd3SPaolo Abeni if (mptcp_pending_data_fin_ack(sk)) { 36816a9a9daSMat Martineau WRITE_ONCE(msk->snd_data_fin_enable, 0); 36916a9a9daSMat Martineau 37016a9a9daSMat Martineau switch (sk->sk_state) { 37116a9a9daSMat Martineau case TCP_FIN_WAIT1: 37216a9a9daSMat Martineau inet_sk_state_store(sk, TCP_FIN_WAIT2); 37316a9a9daSMat Martineau break; 37416a9a9daSMat Martineau case TCP_CLOSING: 37516a9a9daSMat Martineau case TCP_LAST_ACK: 37616a9a9daSMat Martineau inet_sk_state_store(sk, TCP_CLOSE); 37716a9a9daSMat Martineau break; 37816a9a9daSMat Martineau } 37916a9a9daSMat Martineau 380e16163b6SPaolo Abeni mptcp_close_wake_up(sk); 38116a9a9daSMat Martineau } 38216a9a9daSMat Martineau } 38316a9a9daSMat Martineau 3843721b9b6SMat Martineau static bool mptcp_pending_data_fin(struct sock *sk, u64 *seq) 3853721b9b6SMat Martineau { 3863721b9b6SMat Martineau struct mptcp_sock *msk = mptcp_sk(sk); 3873721b9b6SMat Martineau 3883721b9b6SMat Martineau if (READ_ONCE(msk->rcv_data_fin) && 3893721b9b6SMat Martineau ((1 << sk->sk_state) & 3903721b9b6SMat Martineau (TCPF_ESTABLISHED | TCPF_FIN_WAIT1 | TCPF_FIN_WAIT2))) { 3913721b9b6SMat Martineau u64 rcv_data_fin_seq = READ_ONCE(msk->rcv_data_fin_seq); 3923721b9b6SMat Martineau 3933721b9b6SMat Martineau if (msk->ack_seq == rcv_data_fin_seq) { 3943721b9b6SMat Martineau if (seq) 3953721b9b6SMat Martineau *seq = rcv_data_fin_seq; 3963721b9b6SMat Martineau 3973721b9b6SMat Martineau return true; 3983721b9b6SMat Martineau } 3993721b9b6SMat Martineau } 4003721b9b6SMat Martineau 4013721b9b6SMat Martineau return false; 4023721b9b6SMat Martineau } 4033721b9b6SMat Martineau 4043721b9b6SMat Martineau static void mptcp_set_timeout(const struct sock *sk, const struct sock *ssk) 4053721b9b6SMat Martineau { 4063721b9b6SMat Martineau long tout = ssk && inet_csk(ssk)->icsk_pending ? 4073721b9b6SMat Martineau inet_csk(ssk)->icsk_timeout - jiffies : 0; 4083721b9b6SMat Martineau 4093721b9b6SMat Martineau if (tout <= 0) 4103721b9b6SMat Martineau tout = mptcp_sk(sk)->timer_ival; 4113721b9b6SMat Martineau mptcp_sk(sk)->timer_ival = tout > 0 ? tout : TCP_RTO_MIN; 4123721b9b6SMat Martineau } 4133721b9b6SMat Martineau 414ea4ca586SPaolo Abeni static bool mptcp_subflow_active(struct mptcp_subflow_context *subflow) 415ea4ca586SPaolo Abeni { 416ea4ca586SPaolo Abeni struct sock *ssk = mptcp_subflow_tcp_sock(subflow); 417ea4ca586SPaolo Abeni 418ea4ca586SPaolo Abeni /* can't send if JOIN hasn't completed yet (i.e. is usable for mptcp) */ 419ea4ca586SPaolo Abeni if (subflow->request_join && !subflow->fully_established) 420ea4ca586SPaolo Abeni return false; 421ea4ca586SPaolo Abeni 422ea4ca586SPaolo Abeni /* only send if our side has not closed yet */ 423ea4ca586SPaolo Abeni return ((1 << ssk->sk_state) & (TCPF_ESTABLISHED | TCPF_CLOSE_WAIT)); 424ea4ca586SPaolo Abeni } 425ea4ca586SPaolo Abeni 426fd897679SPaolo Abeni static bool tcp_can_send_ack(const struct sock *ssk) 427fd897679SPaolo Abeni { 428fd897679SPaolo Abeni return !((1 << inet_sk_state_load(ssk)) & 42920bc80b6SPaolo Abeni (TCPF_SYN_SENT | TCPF_SYN_RECV | TCPF_TIME_WAIT | TCPF_CLOSE | TCPF_LISTEN)); 430fd897679SPaolo Abeni } 431fd897679SPaolo Abeni 432fd897679SPaolo Abeni static void mptcp_send_ack(struct mptcp_sock *msk) 4337ed90803SPaolo Abeni { 4347ed90803SPaolo Abeni struct mptcp_subflow_context *subflow; 4357ed90803SPaolo Abeni 4367ed90803SPaolo Abeni mptcp_for_each_subflow(msk, subflow) { 4377ed90803SPaolo Abeni struct sock *ssk = mptcp_subflow_tcp_sock(subflow); 4387ed90803SPaolo Abeni 4397ed90803SPaolo Abeni lock_sock(ssk); 440fd897679SPaolo Abeni if (tcp_can_send_ack(ssk)) 4417ed90803SPaolo Abeni tcp_send_ack(ssk); 4427ed90803SPaolo Abeni release_sock(ssk); 443fd897679SPaolo Abeni } 444ea4ca586SPaolo Abeni } 445ea4ca586SPaolo Abeni 446fd897679SPaolo Abeni static bool mptcp_subflow_cleanup_rbuf(struct sock *ssk) 447fd897679SPaolo Abeni { 448fd897679SPaolo Abeni int ret; 449fd897679SPaolo Abeni 450fd897679SPaolo Abeni lock_sock(ssk); 451fd897679SPaolo Abeni ret = tcp_can_send_ack(ssk); 452fd897679SPaolo Abeni if (ret) 453fd897679SPaolo Abeni tcp_cleanup_rbuf(ssk, 1); 454fd897679SPaolo Abeni release_sock(ssk); 455fd897679SPaolo Abeni return ret; 456ea4ca586SPaolo Abeni } 457fd897679SPaolo Abeni 458fd897679SPaolo Abeni static void mptcp_cleanup_rbuf(struct mptcp_sock *msk) 459fd897679SPaolo Abeni { 46087952603SPaolo Abeni struct sock *ack_hint = READ_ONCE(msk->ack_hint); 461e3859603SPaolo Abeni int old_space = READ_ONCE(msk->old_wspace); 462fd897679SPaolo Abeni struct mptcp_subflow_context *subflow; 463e3859603SPaolo Abeni struct sock *sk = (struct sock *)msk; 464e3859603SPaolo Abeni bool cleanup; 465e3859603SPaolo Abeni 466e3859603SPaolo Abeni /* this is a simple superset of what tcp_cleanup_rbuf() implements 467e3859603SPaolo Abeni * so that we don't have to acquire the ssk socket lock most of the time 468e3859603SPaolo Abeni * to do actually nothing 469e3859603SPaolo Abeni */ 470e3859603SPaolo Abeni cleanup = __mptcp_space(sk) - old_space >= max(0, old_space); 471e3859603SPaolo Abeni if (!cleanup) 472e3859603SPaolo Abeni return; 473fd897679SPaolo Abeni 474fd897679SPaolo Abeni /* if the hinted ssk is still active, try to use it */ 47587952603SPaolo Abeni if (likely(ack_hint)) { 476fd897679SPaolo Abeni mptcp_for_each_subflow(msk, subflow) { 477fd897679SPaolo Abeni struct sock *ssk = mptcp_subflow_tcp_sock(subflow); 478fd897679SPaolo Abeni 47987952603SPaolo Abeni if (ack_hint == ssk && mptcp_subflow_cleanup_rbuf(ssk)) 480fd897679SPaolo Abeni return; 4817ed90803SPaolo Abeni } 4827ed90803SPaolo Abeni } 4837ed90803SPaolo Abeni 484fd897679SPaolo Abeni /* otherwise pick the first active subflow */ 485fd897679SPaolo Abeni mptcp_for_each_subflow(msk, subflow) 486fd897679SPaolo Abeni if (mptcp_subflow_cleanup_rbuf(mptcp_subflow_tcp_sock(subflow))) 487fd897679SPaolo Abeni return; 488fd897679SPaolo Abeni } 489fd897679SPaolo Abeni 4907ed90803SPaolo Abeni static bool mptcp_check_data_fin(struct sock *sk) 4913721b9b6SMat Martineau { 4923721b9b6SMat Martineau struct mptcp_sock *msk = mptcp_sk(sk); 4933721b9b6SMat Martineau u64 rcv_data_fin_seq; 4947ed90803SPaolo Abeni bool ret = false; 4953721b9b6SMat Martineau 496*781bf13dSPaolo Abeni if (__mptcp_check_fallback(msk)) 4977ed90803SPaolo Abeni return ret; 4983721b9b6SMat Martineau 4993721b9b6SMat Martineau /* Need to ack a DATA_FIN received from a peer while this side 5003721b9b6SMat Martineau * of the connection is in ESTABLISHED, FIN_WAIT1, or FIN_WAIT2. 5013721b9b6SMat Martineau * msk->rcv_data_fin was set when parsing the incoming options 5023721b9b6SMat Martineau * at the subflow level and the msk lock was not held, so this 5033721b9b6SMat Martineau * is the first opportunity to act on the DATA_FIN and change 5043721b9b6SMat Martineau * the msk state. 5053721b9b6SMat Martineau * 5063721b9b6SMat Martineau * If we are caught up to the sequence number of the incoming 5073721b9b6SMat Martineau * DATA_FIN, send the DATA_ACK now and do state transition. If 5083721b9b6SMat Martineau * not caught up, do nothing and let the recv code send DATA_ACK 5093721b9b6SMat Martineau * when catching up. 5103721b9b6SMat Martineau */ 5113721b9b6SMat Martineau 5123721b9b6SMat Martineau if (mptcp_pending_data_fin(sk, &rcv_data_fin_seq)) { 513917944daSMat Martineau WRITE_ONCE(msk->ack_seq, msk->ack_seq + 1); 5143721b9b6SMat Martineau WRITE_ONCE(msk->rcv_data_fin, 0); 5153721b9b6SMat Martineau 5163721b9b6SMat Martineau sk->sk_shutdown |= RCV_SHUTDOWN; 51716a9a9daSMat Martineau smp_mb__before_atomic(); /* SHUTDOWN must be visible first */ 51816a9a9daSMat Martineau set_bit(MPTCP_DATA_READY, &msk->flags); 5193721b9b6SMat Martineau 5203721b9b6SMat Martineau switch (sk->sk_state) { 5213721b9b6SMat Martineau case TCP_ESTABLISHED: 5223721b9b6SMat Martineau inet_sk_state_store(sk, TCP_CLOSE_WAIT); 5233721b9b6SMat Martineau break; 5243721b9b6SMat Martineau case TCP_FIN_WAIT1: 5253721b9b6SMat Martineau inet_sk_state_store(sk, TCP_CLOSING); 5263721b9b6SMat Martineau break; 5273721b9b6SMat Martineau case TCP_FIN_WAIT2: 5283721b9b6SMat Martineau inet_sk_state_store(sk, TCP_CLOSE); 5293721b9b6SMat Martineau break; 5303721b9b6SMat Martineau default: 5313721b9b6SMat Martineau /* Other states not expected */ 5323721b9b6SMat Martineau WARN_ON_ONCE(1); 5333721b9b6SMat Martineau break; 5343721b9b6SMat Martineau } 5353721b9b6SMat Martineau 5367ed90803SPaolo Abeni ret = true; 5373721b9b6SMat Martineau mptcp_set_timeout(sk, NULL); 538fd897679SPaolo Abeni mptcp_send_ack(msk); 539e16163b6SPaolo Abeni mptcp_close_wake_up(sk); 5403721b9b6SMat Martineau } 5417ed90803SPaolo Abeni return ret; 5423721b9b6SMat Martineau } 5433721b9b6SMat Martineau 5446771bfd9SFlorian Westphal static bool __mptcp_move_skbs_from_subflow(struct mptcp_sock *msk, 5456771bfd9SFlorian Westphal struct sock *ssk, 5466771bfd9SFlorian Westphal unsigned int *bytes) 5476771bfd9SFlorian Westphal { 5486771bfd9SFlorian Westphal struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk); 549600911ffSFlorian Westphal struct sock *sk = (struct sock *)msk; 5506771bfd9SFlorian Westphal unsigned int moved = 0; 5516771bfd9SFlorian Westphal bool more_data_avail; 5526771bfd9SFlorian Westphal struct tcp_sock *tp; 5536771bfd9SFlorian Westphal bool done = false; 55413c7ba0cSFlorian Westphal int sk_rbuf; 55513c7ba0cSFlorian Westphal 55613c7ba0cSFlorian Westphal sk_rbuf = READ_ONCE(sk->sk_rcvbuf); 55713c7ba0cSFlorian Westphal 55813c7ba0cSFlorian Westphal if (!(sk->sk_userlocks & SOCK_RCVBUF_LOCK)) { 55913c7ba0cSFlorian Westphal int ssk_rbuf = READ_ONCE(ssk->sk_rcvbuf); 56013c7ba0cSFlorian Westphal 56113c7ba0cSFlorian Westphal if (unlikely(ssk_rbuf > sk_rbuf)) { 56213c7ba0cSFlorian Westphal WRITE_ONCE(sk->sk_rcvbuf, ssk_rbuf); 56313c7ba0cSFlorian Westphal sk_rbuf = ssk_rbuf; 56413c7ba0cSFlorian Westphal } 56513c7ba0cSFlorian Westphal } 566600911ffSFlorian Westphal 567ab174ad8SPaolo Abeni pr_debug("msk=%p ssk=%p", msk, ssk); 5686771bfd9SFlorian Westphal tp = tcp_sk(ssk); 5696771bfd9SFlorian Westphal do { 5706771bfd9SFlorian Westphal u32 map_remaining, offset; 5716771bfd9SFlorian Westphal u32 seq = tp->copied_seq; 5726771bfd9SFlorian Westphal struct sk_buff *skb; 5736771bfd9SFlorian Westphal bool fin; 5746771bfd9SFlorian Westphal 5756771bfd9SFlorian Westphal /* try to move as much data as available */ 5766771bfd9SFlorian Westphal map_remaining = subflow->map_data_len - 5776771bfd9SFlorian Westphal mptcp_subflow_get_map_offset(subflow); 5786771bfd9SFlorian Westphal 5796771bfd9SFlorian Westphal skb = skb_peek(&ssk->sk_receive_queue); 580d9fb8c50SPaolo Abeni if (!skb) { 581d9fb8c50SPaolo Abeni /* if no data is found, a racing workqueue/recvmsg 582d9fb8c50SPaolo Abeni * already processed the new data, stop here or we 583d9fb8c50SPaolo Abeni * can enter an infinite loop 584d9fb8c50SPaolo Abeni */ 585d9fb8c50SPaolo Abeni if (!moved) 586d9fb8c50SPaolo Abeni done = true; 5876771bfd9SFlorian Westphal break; 588d9fb8c50SPaolo Abeni } 5896771bfd9SFlorian Westphal 590e1ff9e82SDavide Caratti if (__mptcp_check_fallback(msk)) { 591e1ff9e82SDavide Caratti /* if we are running under the workqueue, TCP could have 592e1ff9e82SDavide Caratti * collapsed skbs between dummy map creation and now 593e1ff9e82SDavide Caratti * be sure to adjust the size 594e1ff9e82SDavide Caratti */ 595e1ff9e82SDavide Caratti map_remaining = skb->len; 596e1ff9e82SDavide Caratti subflow->map_data_len = skb->len; 597e1ff9e82SDavide Caratti } 598e1ff9e82SDavide Caratti 5996771bfd9SFlorian Westphal offset = seq - TCP_SKB_CB(skb)->seq; 6006771bfd9SFlorian Westphal fin = TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN; 6016771bfd9SFlorian Westphal if (fin) { 6026771bfd9SFlorian Westphal done = true; 6036771bfd9SFlorian Westphal seq++; 6046771bfd9SFlorian Westphal } 6056771bfd9SFlorian Westphal 6066771bfd9SFlorian Westphal if (offset < skb->len) { 6076771bfd9SFlorian Westphal size_t len = skb->len - offset; 6086771bfd9SFlorian Westphal 6096771bfd9SFlorian Westphal if (tp->urg_data) 6106771bfd9SFlorian Westphal done = true; 6116771bfd9SFlorian Westphal 612ab174ad8SPaolo Abeni if (__mptcp_move_skb(msk, ssk, skb, offset, len)) 6136771bfd9SFlorian Westphal moved += len; 614ab174ad8SPaolo Abeni seq += len; 6156771bfd9SFlorian Westphal 6166771bfd9SFlorian Westphal if (WARN_ON_ONCE(map_remaining < len)) 6176771bfd9SFlorian Westphal break; 6186771bfd9SFlorian Westphal } else { 6196771bfd9SFlorian Westphal WARN_ON_ONCE(!fin); 6206771bfd9SFlorian Westphal sk_eat_skb(ssk, skb); 6216771bfd9SFlorian Westphal done = true; 6226771bfd9SFlorian Westphal } 6236771bfd9SFlorian Westphal 6246771bfd9SFlorian Westphal WRITE_ONCE(tp->copied_seq, seq); 6256771bfd9SFlorian Westphal more_data_avail = mptcp_subflow_data_available(ssk); 626600911ffSFlorian Westphal 62713c7ba0cSFlorian Westphal if (atomic_read(&sk->sk_rmem_alloc) > sk_rbuf) { 628600911ffSFlorian Westphal done = true; 629600911ffSFlorian Westphal break; 630600911ffSFlorian Westphal } 6316771bfd9SFlorian Westphal } while (more_data_avail); 63287952603SPaolo Abeni WRITE_ONCE(msk->ack_hint, ssk); 6336771bfd9SFlorian Westphal 6346719331cSPaolo Abeni *bytes += moved; 6356771bfd9SFlorian Westphal return done; 6366771bfd9SFlorian Westphal } 6376771bfd9SFlorian Westphal 63887952603SPaolo Abeni static bool __mptcp_ofo_queue(struct mptcp_sock *msk) 639ab174ad8SPaolo Abeni { 640ab174ad8SPaolo Abeni struct sock *sk = (struct sock *)msk; 641ab174ad8SPaolo Abeni struct sk_buff *skb, *tail; 642ab174ad8SPaolo Abeni bool moved = false; 643ab174ad8SPaolo Abeni struct rb_node *p; 644ab174ad8SPaolo Abeni u64 end_seq; 645ab174ad8SPaolo Abeni 646ab174ad8SPaolo Abeni p = rb_first(&msk->out_of_order_queue); 64706242e44SPaolo Abeni pr_debug("msk=%p empty=%d", msk, RB_EMPTY_ROOT(&msk->out_of_order_queue)); 648ab174ad8SPaolo Abeni while (p) { 649ab174ad8SPaolo Abeni skb = rb_to_skb(p); 650ab174ad8SPaolo Abeni if (after64(MPTCP_SKB_CB(skb)->map_seq, msk->ack_seq)) 651ab174ad8SPaolo Abeni break; 652ab174ad8SPaolo Abeni 653ab174ad8SPaolo Abeni p = rb_next(p); 654ab174ad8SPaolo Abeni rb_erase(&skb->rbnode, &msk->out_of_order_queue); 655ab174ad8SPaolo Abeni 656ab174ad8SPaolo Abeni if (unlikely(!after64(MPTCP_SKB_CB(skb)->end_seq, 657ab174ad8SPaolo Abeni msk->ack_seq))) { 658ab174ad8SPaolo Abeni mptcp_drop(sk, skb); 65906242e44SPaolo Abeni MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_DUPDATA); 660ab174ad8SPaolo Abeni continue; 661ab174ad8SPaolo Abeni } 662ab174ad8SPaolo Abeni 663ab174ad8SPaolo Abeni end_seq = MPTCP_SKB_CB(skb)->end_seq; 664ab174ad8SPaolo Abeni tail = skb_peek_tail(&sk->sk_receive_queue); 665ab174ad8SPaolo Abeni if (!tail || !mptcp_ooo_try_coalesce(msk, tail, skb)) { 666ab174ad8SPaolo Abeni int delta = msk->ack_seq - MPTCP_SKB_CB(skb)->map_seq; 667ab174ad8SPaolo Abeni 668ab174ad8SPaolo Abeni /* skip overlapping data, if any */ 66906242e44SPaolo Abeni pr_debug("uncoalesced seq=%llx ack seq=%llx delta=%d", 67006242e44SPaolo Abeni MPTCP_SKB_CB(skb)->map_seq, msk->ack_seq, 67106242e44SPaolo Abeni delta); 672ab174ad8SPaolo Abeni MPTCP_SKB_CB(skb)->offset += delta; 673ab174ad8SPaolo Abeni __skb_queue_tail(&sk->sk_receive_queue, skb); 674ab174ad8SPaolo Abeni } 675ab174ad8SPaolo Abeni msk->ack_seq = end_seq; 676ab174ad8SPaolo Abeni moved = true; 677ab174ad8SPaolo Abeni } 678ab174ad8SPaolo Abeni return moved; 679ab174ad8SPaolo Abeni } 680ab174ad8SPaolo Abeni 6812e52213cSFlorian Westphal /* In most cases we will be able to lock the mptcp socket. If its already 6822e52213cSFlorian Westphal * owned, we need to defer to the work queue to avoid ABBA deadlock. 6832e52213cSFlorian Westphal */ 68487952603SPaolo Abeni static void move_skbs_to_msk(struct mptcp_sock *msk, struct sock *ssk) 6852e52213cSFlorian Westphal { 6862e52213cSFlorian Westphal struct sock *sk = (struct sock *)msk; 6872e52213cSFlorian Westphal unsigned int moved = 0; 6882e52213cSFlorian Westphal 68987952603SPaolo Abeni if (inet_sk_state_load(sk) == TCP_CLOSE) 69087952603SPaolo Abeni return; 6912e52213cSFlorian Westphal 69287952603SPaolo Abeni mptcp_data_lock(sk); 6932e52213cSFlorian Westphal 6942e52213cSFlorian Westphal __mptcp_move_skbs_from_subflow(msk, ssk, &moved); 69587952603SPaolo Abeni __mptcp_ofo_queue(msk); 696ab174ad8SPaolo Abeni 697ab174ad8SPaolo Abeni /* If the moves have caught up with the DATA_FIN sequence number 698ab174ad8SPaolo Abeni * it's time to ack the DATA_FIN and change socket state, but 699ab174ad8SPaolo Abeni * this is not a good place to change state. Let the workqueue 700ab174ad8SPaolo Abeni * do it. 701ab174ad8SPaolo Abeni */ 702ba8f48f7SPaolo Abeni if (mptcp_pending_data_fin(sk, NULL)) 703ba8f48f7SPaolo Abeni mptcp_schedule_work(sk); 70487952603SPaolo Abeni mptcp_data_unlock(sk); 7052e52213cSFlorian Westphal } 7062e52213cSFlorian Westphal 7072e52213cSFlorian Westphal void mptcp_data_ready(struct sock *sk, struct sock *ssk) 708101f6f85SFlorian Westphal { 7096719331cSPaolo Abeni struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk); 710101f6f85SFlorian Westphal struct mptcp_sock *msk = mptcp_sk(sk); 71113c7ba0cSFlorian Westphal int sk_rbuf, ssk_rbuf; 7126719331cSPaolo Abeni bool wake; 713101f6f85SFlorian Westphal 714d7b1bfd0SPaolo Abeni /* The peer can send data while we are shutting down this 715d7b1bfd0SPaolo Abeni * subflow at msk destruction time, but we must avoid enqueuing 716d7b1bfd0SPaolo Abeni * more data to the msk receive queue 717d7b1bfd0SPaolo Abeni */ 718d7b1bfd0SPaolo Abeni if (unlikely(subflow->disposable)) 719d7b1bfd0SPaolo Abeni return; 720d7b1bfd0SPaolo Abeni 7216719331cSPaolo Abeni /* move_skbs_to_msk below can legitly clear the data_avail flag, 7226719331cSPaolo Abeni * but we will need later to properly woke the reader, cache its 7236719331cSPaolo Abeni * value 7246719331cSPaolo Abeni */ 7256719331cSPaolo Abeni wake = subflow->data_avail == MPTCP_SUBFLOW_DATA_AVAIL; 7266719331cSPaolo Abeni if (wake) 727101f6f85SFlorian Westphal set_bit(MPTCP_DATA_READY, &msk->flags); 7286771bfd9SFlorian Westphal 72913c7ba0cSFlorian Westphal ssk_rbuf = READ_ONCE(ssk->sk_rcvbuf); 73013c7ba0cSFlorian Westphal sk_rbuf = READ_ONCE(sk->sk_rcvbuf); 73113c7ba0cSFlorian Westphal if (unlikely(ssk_rbuf > sk_rbuf)) 73213c7ba0cSFlorian Westphal sk_rbuf = ssk_rbuf; 73313c7ba0cSFlorian Westphal 73413c7ba0cSFlorian Westphal /* over limit? can't append more skbs to msk */ 73513c7ba0cSFlorian Westphal if (atomic_read(&sk->sk_rmem_alloc) > sk_rbuf) 7362e52213cSFlorian Westphal goto wake; 7372e52213cSFlorian Westphal 73814c441b5SPaolo Abeni move_skbs_to_msk(msk, ssk); 739ea4ca586SPaolo Abeni 740600911ffSFlorian Westphal wake: 7416719331cSPaolo Abeni if (wake) 742101f6f85SFlorian Westphal sk->sk_data_ready(sk); 743101f6f85SFlorian Westphal } 744101f6f85SFlorian Westphal 74584dfe367SGeliang Tang void __mptcp_flush_join_list(struct mptcp_sock *msk) 746ec3edaa7SPeter Krystad { 7475cf92bbaSPaolo Abeni struct mptcp_subflow_context *subflow; 7485cf92bbaSPaolo Abeni 749ec3edaa7SPeter Krystad if (likely(list_empty(&msk->join_list))) 750ec3edaa7SPeter Krystad return; 751ec3edaa7SPeter Krystad 752ec3edaa7SPeter Krystad spin_lock_bh(&msk->join_list_lock); 7535cf92bbaSPaolo Abeni list_for_each_entry(subflow, &msk->join_list, node) 7545cf92bbaSPaolo Abeni mptcp_propagate_sndbuf((struct sock *)msk, mptcp_subflow_tcp_sock(subflow)); 755ec3edaa7SPeter Krystad list_splice_tail_init(&msk->join_list, &msk->conn_list); 756ec3edaa7SPeter Krystad spin_unlock_bh(&msk->join_list_lock); 757ec3edaa7SPeter Krystad } 758ec3edaa7SPeter Krystad 759b51f9b80SPaolo Abeni static bool mptcp_timer_pending(struct sock *sk) 760b51f9b80SPaolo Abeni { 761b51f9b80SPaolo Abeni return timer_pending(&inet_csk(sk)->icsk_retransmit_timer); 762b51f9b80SPaolo Abeni } 763b51f9b80SPaolo Abeni 764b51f9b80SPaolo Abeni static void mptcp_reset_timer(struct sock *sk) 765b51f9b80SPaolo Abeni { 766b51f9b80SPaolo Abeni struct inet_connection_sock *icsk = inet_csk(sk); 767b51f9b80SPaolo Abeni unsigned long tout; 768b51f9b80SPaolo Abeni 769e16163b6SPaolo Abeni /* prevent rescheduling on close */ 770e16163b6SPaolo Abeni if (unlikely(inet_sk_state_load(sk) == TCP_CLOSE)) 771e16163b6SPaolo Abeni return; 772e16163b6SPaolo Abeni 773b51f9b80SPaolo Abeni /* should never be called with mptcp level timer cleared */ 774b51f9b80SPaolo Abeni tout = READ_ONCE(mptcp_sk(sk)->timer_ival); 775b51f9b80SPaolo Abeni if (WARN_ON_ONCE(!tout)) 776b51f9b80SPaolo Abeni tout = TCP_RTO_MIN; 777b51f9b80SPaolo Abeni sk_reset_timer(sk, &icsk->icsk_retransmit_timer, jiffies + tout); 778b51f9b80SPaolo Abeni } 779b51f9b80SPaolo Abeni 780ba8f48f7SPaolo Abeni bool mptcp_schedule_work(struct sock *sk) 781ba8f48f7SPaolo Abeni { 782ba8f48f7SPaolo Abeni if (inet_sk_state_load(sk) != TCP_CLOSE && 783ba8f48f7SPaolo Abeni schedule_work(&mptcp_sk(sk)->work)) { 784ba8f48f7SPaolo Abeni /* each subflow already holds a reference to the sk, and the 785ba8f48f7SPaolo Abeni * workqueue is invoked by a subflow, so sk can't go away here. 786ba8f48f7SPaolo Abeni */ 787ba8f48f7SPaolo Abeni sock_hold(sk); 788ba8f48f7SPaolo Abeni return true; 789ba8f48f7SPaolo Abeni } 790ba8f48f7SPaolo Abeni return false; 791ba8f48f7SPaolo Abeni } 792ba8f48f7SPaolo Abeni 79359832e24SFlorian Westphal void mptcp_subflow_eof(struct sock *sk) 79459832e24SFlorian Westphal { 795ba8f48f7SPaolo Abeni if (!test_and_set_bit(MPTCP_WORK_EOF, &mptcp_sk(sk)->flags)) 796ba8f48f7SPaolo Abeni mptcp_schedule_work(sk); 79759832e24SFlorian Westphal } 79859832e24SFlorian Westphal 7995969856aSPaolo Abeni static void mptcp_check_for_eof(struct mptcp_sock *msk) 8005969856aSPaolo Abeni { 8015969856aSPaolo Abeni struct mptcp_subflow_context *subflow; 8025969856aSPaolo Abeni struct sock *sk = (struct sock *)msk; 8035969856aSPaolo Abeni int receivers = 0; 8045969856aSPaolo Abeni 8055969856aSPaolo Abeni mptcp_for_each_subflow(msk, subflow) 8065969856aSPaolo Abeni receivers += !subflow->rx_eof; 807e16163b6SPaolo Abeni if (receivers) 808e16163b6SPaolo Abeni return; 8095969856aSPaolo Abeni 810e16163b6SPaolo Abeni if (!(sk->sk_shutdown & RCV_SHUTDOWN)) { 8115969856aSPaolo Abeni /* hopefully temporary hack: propagate shutdown status 8125969856aSPaolo Abeni * to msk, when all subflows agree on it 8135969856aSPaolo Abeni */ 8145969856aSPaolo Abeni sk->sk_shutdown |= RCV_SHUTDOWN; 8155969856aSPaolo Abeni 8165969856aSPaolo Abeni smp_mb__before_atomic(); /* SHUTDOWN must be visible first */ 8175969856aSPaolo Abeni set_bit(MPTCP_DATA_READY, &msk->flags); 8185969856aSPaolo Abeni sk->sk_data_ready(sk); 8195969856aSPaolo Abeni } 820e16163b6SPaolo Abeni 821e16163b6SPaolo Abeni switch (sk->sk_state) { 822e16163b6SPaolo Abeni case TCP_ESTABLISHED: 823e16163b6SPaolo Abeni inet_sk_state_store(sk, TCP_CLOSE_WAIT); 824e16163b6SPaolo Abeni break; 825e16163b6SPaolo Abeni case TCP_FIN_WAIT1: 82626aa2314SPaolo Abeni inet_sk_state_store(sk, TCP_CLOSING); 82726aa2314SPaolo Abeni break; 82826aa2314SPaolo Abeni case TCP_FIN_WAIT2: 829e16163b6SPaolo Abeni inet_sk_state_store(sk, TCP_CLOSE); 830e16163b6SPaolo Abeni break; 831e16163b6SPaolo Abeni default: 832e16163b6SPaolo Abeni return; 833e16163b6SPaolo Abeni } 834e16163b6SPaolo Abeni mptcp_close_wake_up(sk); 8355969856aSPaolo Abeni } 8365969856aSPaolo Abeni 8377a6a6cbcSPaolo Abeni static struct sock *mptcp_subflow_recv_lookup(const struct mptcp_sock *msk) 8387a6a6cbcSPaolo Abeni { 8397a6a6cbcSPaolo Abeni struct mptcp_subflow_context *subflow; 8407a6a6cbcSPaolo Abeni struct sock *sk = (struct sock *)msk; 8417a6a6cbcSPaolo Abeni 8427a6a6cbcSPaolo Abeni sock_owned_by_me(sk); 8437a6a6cbcSPaolo Abeni 8447a6a6cbcSPaolo Abeni mptcp_for_each_subflow(msk, subflow) { 8457a6a6cbcSPaolo Abeni if (subflow->data_avail) 8467a6a6cbcSPaolo Abeni return mptcp_subflow_tcp_sock(subflow); 8477a6a6cbcSPaolo Abeni } 8487a6a6cbcSPaolo Abeni 8497a6a6cbcSPaolo Abeni return NULL; 8507a6a6cbcSPaolo Abeni } 8517a6a6cbcSPaolo Abeni 8523f8e0aaeSPaolo Abeni static bool mptcp_skb_can_collapse_to(u64 write_seq, 85357040755SPaolo Abeni const struct sk_buff *skb, 85457040755SPaolo Abeni const struct mptcp_ext *mpext) 8556d0060f6SMat Martineau { 85657040755SPaolo Abeni if (!tcp_skb_can_collapse_to(skb)) 85757040755SPaolo Abeni return false; 85857040755SPaolo Abeni 8595a369ca6SPaolo Abeni /* can collapse only if MPTCP level sequence is in order and this 8605a369ca6SPaolo Abeni * mapping has not been xmitted yet 8615a369ca6SPaolo Abeni */ 8625a369ca6SPaolo Abeni return mpext && mpext->data_seq + mpext->data_len == write_seq && 8635a369ca6SPaolo Abeni !mpext->frozen; 86457040755SPaolo Abeni } 86557040755SPaolo Abeni 86618b683bfSPaolo Abeni static bool mptcp_frag_can_collapse_to(const struct mptcp_sock *msk, 86718b683bfSPaolo Abeni const struct page_frag *pfrag, 86818b683bfSPaolo Abeni const struct mptcp_data_frag *df) 86918b683bfSPaolo Abeni { 87018b683bfSPaolo Abeni return df && pfrag->page == df->page && 871d9ca1de8SPaolo Abeni pfrag->size - pfrag->offset > 0 && 87218b683bfSPaolo Abeni df->data_seq + df->data_len == msk->write_seq; 87318b683bfSPaolo Abeni } 87418b683bfSPaolo Abeni 875724cfd2eSPaolo Abeni static int mptcp_wmem_with_overhead(struct sock *sk, int size) 876e93da928SPaolo Abeni { 877724cfd2eSPaolo Abeni struct mptcp_sock *msk = mptcp_sk(sk); 878724cfd2eSPaolo Abeni int ret, skbs; 879724cfd2eSPaolo Abeni 880724cfd2eSPaolo Abeni ret = size + ((sizeof(struct mptcp_data_frag) * size) >> PAGE_SHIFT); 881724cfd2eSPaolo Abeni skbs = (msk->tx_pending_data + size) / msk->size_goal_cache; 882724cfd2eSPaolo Abeni if (skbs < msk->skb_tx_cache.qlen) 883724cfd2eSPaolo Abeni return ret; 884724cfd2eSPaolo Abeni 885724cfd2eSPaolo Abeni return ret + (skbs - msk->skb_tx_cache.qlen) * SKB_TRUESIZE(MAX_TCP_HEADER); 886e93da928SPaolo Abeni } 887e93da928SPaolo Abeni 888e93da928SPaolo Abeni static void __mptcp_wmem_reserve(struct sock *sk, int size) 889e93da928SPaolo Abeni { 890724cfd2eSPaolo Abeni int amount = mptcp_wmem_with_overhead(sk, size); 891e93da928SPaolo Abeni struct mptcp_sock *msk = mptcp_sk(sk); 892e93da928SPaolo Abeni 893e93da928SPaolo Abeni WARN_ON_ONCE(msk->wmem_reserved); 894e7579d5dSDavide Caratti if (WARN_ON_ONCE(amount < 0)) 895e7579d5dSDavide Caratti amount = 0; 896e7579d5dSDavide Caratti 897e93da928SPaolo Abeni if (amount <= sk->sk_forward_alloc) 898e93da928SPaolo Abeni goto reserve; 899e93da928SPaolo Abeni 900e93da928SPaolo Abeni /* under memory pressure try to reserve at most a single page 901e93da928SPaolo Abeni * otherwise try to reserve the full estimate and fallback 902e93da928SPaolo Abeni * to a single page before entering the error path 903e93da928SPaolo Abeni */ 904e93da928SPaolo Abeni if ((tcp_under_memory_pressure(sk) && amount > PAGE_SIZE) || 905e93da928SPaolo Abeni !sk_wmem_schedule(sk, amount)) { 906e93da928SPaolo Abeni if (amount <= PAGE_SIZE) 907e93da928SPaolo Abeni goto nomem; 908e93da928SPaolo Abeni 909e93da928SPaolo Abeni amount = PAGE_SIZE; 910e93da928SPaolo Abeni if (!sk_wmem_schedule(sk, amount)) 911e93da928SPaolo Abeni goto nomem; 912e93da928SPaolo Abeni } 913e93da928SPaolo Abeni 914e93da928SPaolo Abeni reserve: 915e93da928SPaolo Abeni msk->wmem_reserved = amount; 916e93da928SPaolo Abeni sk->sk_forward_alloc -= amount; 917e93da928SPaolo Abeni return; 918e93da928SPaolo Abeni 919e93da928SPaolo Abeni nomem: 920e93da928SPaolo Abeni /* we will wait for memory on next allocation */ 921e93da928SPaolo Abeni msk->wmem_reserved = -1; 922e93da928SPaolo Abeni } 923e93da928SPaolo Abeni 924e93da928SPaolo Abeni static void __mptcp_update_wmem(struct sock *sk) 925e93da928SPaolo Abeni { 926e93da928SPaolo Abeni struct mptcp_sock *msk = mptcp_sk(sk); 927e93da928SPaolo Abeni 928e93da928SPaolo Abeni if (!msk->wmem_reserved) 929e93da928SPaolo Abeni return; 930e93da928SPaolo Abeni 931e93da928SPaolo Abeni if (msk->wmem_reserved < 0) 932e93da928SPaolo Abeni msk->wmem_reserved = 0; 933e93da928SPaolo Abeni if (msk->wmem_reserved > 0) { 934e93da928SPaolo Abeni sk->sk_forward_alloc += msk->wmem_reserved; 935e93da928SPaolo Abeni msk->wmem_reserved = 0; 936e93da928SPaolo Abeni } 937e93da928SPaolo Abeni } 938e93da928SPaolo Abeni 939e93da928SPaolo Abeni static bool mptcp_wmem_alloc(struct sock *sk, int size) 940e93da928SPaolo Abeni { 941e93da928SPaolo Abeni struct mptcp_sock *msk = mptcp_sk(sk); 942e93da928SPaolo Abeni 943e93da928SPaolo Abeni /* check for pre-existing error condition */ 944e93da928SPaolo Abeni if (msk->wmem_reserved < 0) 945e93da928SPaolo Abeni return false; 946e93da928SPaolo Abeni 947e93da928SPaolo Abeni if (msk->wmem_reserved >= size) 948e93da928SPaolo Abeni goto account; 949e93da928SPaolo Abeni 95087952603SPaolo Abeni mptcp_data_lock(sk); 95187952603SPaolo Abeni if (!sk_wmem_schedule(sk, size)) { 95287952603SPaolo Abeni mptcp_data_unlock(sk); 953e93da928SPaolo Abeni return false; 95487952603SPaolo Abeni } 955e93da928SPaolo Abeni 956e93da928SPaolo Abeni sk->sk_forward_alloc -= size; 957e93da928SPaolo Abeni msk->wmem_reserved += size; 95887952603SPaolo Abeni mptcp_data_unlock(sk); 959e93da928SPaolo Abeni 960e93da928SPaolo Abeni account: 961e93da928SPaolo Abeni msk->wmem_reserved -= size; 962e93da928SPaolo Abeni return true; 963e93da928SPaolo Abeni } 964e93da928SPaolo Abeni 96587952603SPaolo Abeni static void mptcp_wmem_uncharge(struct sock *sk, int size) 96687952603SPaolo Abeni { 96787952603SPaolo Abeni struct mptcp_sock *msk = mptcp_sk(sk); 96887952603SPaolo Abeni 96987952603SPaolo Abeni if (msk->wmem_reserved < 0) 97087952603SPaolo Abeni msk->wmem_reserved = 0; 97187952603SPaolo Abeni msk->wmem_reserved += size; 97287952603SPaolo Abeni } 97387952603SPaolo Abeni 974724cfd2eSPaolo Abeni static void mptcp_mem_reclaim_partial(struct sock *sk) 975724cfd2eSPaolo Abeni { 976724cfd2eSPaolo Abeni struct mptcp_sock *msk = mptcp_sk(sk); 977724cfd2eSPaolo Abeni 978724cfd2eSPaolo Abeni /* if we are experiencing a transint allocation error, 979724cfd2eSPaolo Abeni * the forward allocation memory has been already 980724cfd2eSPaolo Abeni * released 981724cfd2eSPaolo Abeni */ 982724cfd2eSPaolo Abeni if (msk->wmem_reserved < 0) 983724cfd2eSPaolo Abeni return; 984724cfd2eSPaolo Abeni 985724cfd2eSPaolo Abeni mptcp_data_lock(sk); 986724cfd2eSPaolo Abeni sk->sk_forward_alloc += msk->wmem_reserved; 987724cfd2eSPaolo Abeni sk_mem_reclaim_partial(sk); 988724cfd2eSPaolo Abeni msk->wmem_reserved = sk->sk_forward_alloc; 989724cfd2eSPaolo Abeni sk->sk_forward_alloc = 0; 990724cfd2eSPaolo Abeni mptcp_data_unlock(sk); 991724cfd2eSPaolo Abeni } 992724cfd2eSPaolo Abeni 993d027236cSPaolo Abeni static void dfrag_uncharge(struct sock *sk, int len) 99418b683bfSPaolo Abeni { 995d027236cSPaolo Abeni sk_mem_uncharge(sk, len); 9967948f6ccSFlorian Westphal sk_wmem_queued_add(sk, -len); 997d027236cSPaolo Abeni } 998d027236cSPaolo Abeni 999d027236cSPaolo Abeni static void dfrag_clear(struct sock *sk, struct mptcp_data_frag *dfrag) 1000d027236cSPaolo Abeni { 1001d027236cSPaolo Abeni int len = dfrag->data_len + dfrag->overhead; 1002d027236cSPaolo Abeni 100318b683bfSPaolo Abeni list_del(&dfrag->list); 1004d027236cSPaolo Abeni dfrag_uncharge(sk, len); 100518b683bfSPaolo Abeni put_page(dfrag->page); 100618b683bfSPaolo Abeni } 100718b683bfSPaolo Abeni 10086e628cd3SPaolo Abeni static void __mptcp_clean_una(struct sock *sk) 100918b683bfSPaolo Abeni { 101018b683bfSPaolo Abeni struct mptcp_sock *msk = mptcp_sk(sk); 101118b683bfSPaolo Abeni struct mptcp_data_frag *dtmp, *dfrag; 1012d027236cSPaolo Abeni bool cleaned = false; 1013e1ff9e82SDavide Caratti u64 snd_una; 1014e1ff9e82SDavide Caratti 1015e1ff9e82SDavide Caratti /* on fallback we just need to ignore snd_una, as this is really 1016e1ff9e82SDavide Caratti * plain TCP 1017e1ff9e82SDavide Caratti */ 1018e1ff9e82SDavide Caratti if (__mptcp_check_fallback(msk)) 10197439d687SPaolo Abeni msk->snd_una = READ_ONCE(msk->snd_nxt); 10207439d687SPaolo Abeni 10217439d687SPaolo Abeni snd_una = msk->snd_una; 102218b683bfSPaolo Abeni list_for_each_entry_safe(dfrag, dtmp, &msk->rtx_queue, list) { 102318b683bfSPaolo Abeni if (after64(dfrag->data_seq + dfrag->data_len, snd_una)) 102418b683bfSPaolo Abeni break; 102518b683bfSPaolo Abeni 1026d9ca1de8SPaolo Abeni if (WARN_ON_ONCE(dfrag == msk->first_pending)) 1027d9ca1de8SPaolo Abeni break; 1028d027236cSPaolo Abeni dfrag_clear(sk, dfrag); 1029d027236cSPaolo Abeni cleaned = true; 1030d027236cSPaolo Abeni } 1031d027236cSPaolo Abeni 10327948f6ccSFlorian Westphal dfrag = mptcp_rtx_head(sk); 10337948f6ccSFlorian Westphal if (dfrag && after64(snd_una, dfrag->data_seq)) { 103453eb4c38SPaolo Abeni u64 delta = snd_una - dfrag->data_seq; 103553eb4c38SPaolo Abeni 1036d9ca1de8SPaolo Abeni if (WARN_ON_ONCE(delta > dfrag->already_sent)) 103753eb4c38SPaolo Abeni goto out; 10387948f6ccSFlorian Westphal 10397948f6ccSFlorian Westphal dfrag->data_seq += delta; 104053eb4c38SPaolo Abeni dfrag->offset += delta; 10417948f6ccSFlorian Westphal dfrag->data_len -= delta; 1042d9ca1de8SPaolo Abeni dfrag->already_sent -= delta; 10437948f6ccSFlorian Westphal 10447948f6ccSFlorian Westphal dfrag_uncharge(sk, delta); 10457948f6ccSFlorian Westphal cleaned = true; 10467948f6ccSFlorian Westphal } 10477948f6ccSFlorian Westphal 104853eb4c38SPaolo Abeni out: 10496e628cd3SPaolo Abeni if (cleaned) { 10506e628cd3SPaolo Abeni if (tcp_under_memory_pressure(sk)) { 10516e628cd3SPaolo Abeni __mptcp_update_wmem(sk); 1052d027236cSPaolo Abeni sk_mem_reclaim_partial(sk); 105395ed690eSFlorian Westphal } 105418b683bfSPaolo Abeni } 105518b683bfSPaolo Abeni 10566e628cd3SPaolo Abeni if (snd_una == READ_ONCE(msk->snd_nxt)) { 10576e628cd3SPaolo Abeni if (msk->timer_ival) 10586e628cd3SPaolo Abeni mptcp_stop_timer(sk); 10596e628cd3SPaolo Abeni } else { 10606e628cd3SPaolo Abeni mptcp_reset_timer(sk); 10616e628cd3SPaolo Abeni } 10626e628cd3SPaolo Abeni } 10636e628cd3SPaolo Abeni 1064417789dfSPaolo Abeni static void __mptcp_clean_una_wakeup(struct sock *sk) 1065417789dfSPaolo Abeni { 1066417789dfSPaolo Abeni __mptcp_clean_una(sk); 1067417789dfSPaolo Abeni mptcp_write_space(sk); 1068417789dfSPaolo Abeni } 1069417789dfSPaolo Abeni 1070724cfd2eSPaolo Abeni static void mptcp_enter_memory_pressure(struct sock *sk) 107118b683bfSPaolo Abeni { 1072d9ca1de8SPaolo Abeni struct mptcp_subflow_context *subflow; 1073d9ca1de8SPaolo Abeni struct mptcp_sock *msk = mptcp_sk(sk); 1074d9ca1de8SPaolo Abeni bool first = true; 1075d9ca1de8SPaolo Abeni 107618b683bfSPaolo Abeni sk_stream_moderate_sndbuf(sk); 1077d9ca1de8SPaolo Abeni mptcp_for_each_subflow(msk, subflow) { 1078d9ca1de8SPaolo Abeni struct sock *ssk = mptcp_subflow_tcp_sock(subflow); 1079d9ca1de8SPaolo Abeni 1080d9ca1de8SPaolo Abeni if (first) 1081d9ca1de8SPaolo Abeni tcp_enter_memory_pressure(ssk); 1082d9ca1de8SPaolo Abeni sk_stream_moderate_sndbuf(ssk); 1083d9ca1de8SPaolo Abeni first = false; 1084d9ca1de8SPaolo Abeni } 1085724cfd2eSPaolo Abeni } 1086724cfd2eSPaolo Abeni 1087724cfd2eSPaolo Abeni /* ensure we get enough memory for the frag hdr, beyond some minimal amount of 1088724cfd2eSPaolo Abeni * data 1089724cfd2eSPaolo Abeni */ 1090724cfd2eSPaolo Abeni static bool mptcp_page_frag_refill(struct sock *sk, struct page_frag *pfrag) 1091724cfd2eSPaolo Abeni { 1092724cfd2eSPaolo Abeni if (likely(skb_page_frag_refill(32U + sizeof(struct mptcp_data_frag), 1093724cfd2eSPaolo Abeni pfrag, sk->sk_allocation))) 1094724cfd2eSPaolo Abeni return true; 1095724cfd2eSPaolo Abeni 1096724cfd2eSPaolo Abeni mptcp_enter_memory_pressure(sk); 109718b683bfSPaolo Abeni return false; 109818b683bfSPaolo Abeni } 109918b683bfSPaolo Abeni 110018b683bfSPaolo Abeni static struct mptcp_data_frag * 110118b683bfSPaolo Abeni mptcp_carve_data_frag(const struct mptcp_sock *msk, struct page_frag *pfrag, 110218b683bfSPaolo Abeni int orig_offset) 110318b683bfSPaolo Abeni { 110418b683bfSPaolo Abeni int offset = ALIGN(orig_offset, sizeof(long)); 110518b683bfSPaolo Abeni struct mptcp_data_frag *dfrag; 110618b683bfSPaolo Abeni 110718b683bfSPaolo Abeni dfrag = (struct mptcp_data_frag *)(page_to_virt(pfrag->page) + offset); 110818b683bfSPaolo Abeni dfrag->data_len = 0; 110918b683bfSPaolo Abeni dfrag->data_seq = msk->write_seq; 111018b683bfSPaolo Abeni dfrag->overhead = offset - orig_offset + sizeof(struct mptcp_data_frag); 111118b683bfSPaolo Abeni dfrag->offset = offset + sizeof(struct mptcp_data_frag); 1112d9ca1de8SPaolo Abeni dfrag->already_sent = 0; 111318b683bfSPaolo Abeni dfrag->page = pfrag->page; 111418b683bfSPaolo Abeni 111518b683bfSPaolo Abeni return dfrag; 111618b683bfSPaolo Abeni } 111718b683bfSPaolo Abeni 1118caf971dfSPaolo Abeni struct mptcp_sendmsg_info { 1119caf971dfSPaolo Abeni int mss_now; 1120caf971dfSPaolo Abeni int size_goal; 1121d9ca1de8SPaolo Abeni u16 limit; 1122d9ca1de8SPaolo Abeni u16 sent; 1123d9ca1de8SPaolo Abeni unsigned int flags; 1124caf971dfSPaolo Abeni }; 1125caf971dfSPaolo Abeni 11266f8a612aSFlorian Westphal static int mptcp_check_allowed_size(struct mptcp_sock *msk, u64 data_seq, 11276f8a612aSFlorian Westphal int avail_size) 11286f8a612aSFlorian Westphal { 11296f8a612aSFlorian Westphal u64 window_end = mptcp_wnd_end(msk); 11306f8a612aSFlorian Westphal 11316f8a612aSFlorian Westphal if (__mptcp_check_fallback(msk)) 11326f8a612aSFlorian Westphal return avail_size; 11336f8a612aSFlorian Westphal 11346f8a612aSFlorian Westphal if (!before64(data_seq + avail_size, window_end)) { 11356f8a612aSFlorian Westphal u64 allowed_size = window_end - data_seq; 11366f8a612aSFlorian Westphal 11376f8a612aSFlorian Westphal return min_t(unsigned int, allowed_size, avail_size); 11386f8a612aSFlorian Westphal } 11396f8a612aSFlorian Westphal 11406f8a612aSFlorian Westphal return avail_size; 11416f8a612aSFlorian Westphal } 11426f8a612aSFlorian Westphal 1143724cfd2eSPaolo Abeni static bool __mptcp_add_ext(struct sk_buff *skb, gfp_t gfp) 1144724cfd2eSPaolo Abeni { 1145724cfd2eSPaolo Abeni struct skb_ext *mpext = __skb_ext_alloc(gfp); 1146724cfd2eSPaolo Abeni 1147724cfd2eSPaolo Abeni if (!mpext) 1148724cfd2eSPaolo Abeni return false; 1149724cfd2eSPaolo Abeni __skb_ext_set(skb, SKB_EXT_MPTCP, mpext); 1150724cfd2eSPaolo Abeni return true; 1151724cfd2eSPaolo Abeni } 1152724cfd2eSPaolo Abeni 11536e628cd3SPaolo Abeni static struct sk_buff *__mptcp_do_alloc_tx_skb(struct sock *sk, gfp_t gfp) 1154724cfd2eSPaolo Abeni { 1155724cfd2eSPaolo Abeni struct sk_buff *skb; 1156724cfd2eSPaolo Abeni 11576e628cd3SPaolo Abeni skb = alloc_skb_fclone(MAX_TCP_HEADER, gfp); 1158724cfd2eSPaolo Abeni if (likely(skb)) { 11596e628cd3SPaolo Abeni if (likely(__mptcp_add_ext(skb, gfp))) { 1160724cfd2eSPaolo Abeni skb_reserve(skb, MAX_TCP_HEADER); 1161724cfd2eSPaolo Abeni skb->reserved_tailroom = skb->end - skb->tail; 1162724cfd2eSPaolo Abeni return skb; 1163724cfd2eSPaolo Abeni } 1164724cfd2eSPaolo Abeni __kfree_skb(skb); 1165724cfd2eSPaolo Abeni } else { 1166724cfd2eSPaolo Abeni mptcp_enter_memory_pressure(sk); 1167724cfd2eSPaolo Abeni } 1168724cfd2eSPaolo Abeni return NULL; 1169724cfd2eSPaolo Abeni } 1170724cfd2eSPaolo Abeni 1171724cfd2eSPaolo Abeni static bool mptcp_tx_cache_refill(struct sock *sk, int size, 1172724cfd2eSPaolo Abeni struct sk_buff_head *skbs, int *total_ts) 1173724cfd2eSPaolo Abeni { 1174724cfd2eSPaolo Abeni struct mptcp_sock *msk = mptcp_sk(sk); 1175724cfd2eSPaolo Abeni struct sk_buff *skb; 1176724cfd2eSPaolo Abeni int space_needed; 1177724cfd2eSPaolo Abeni 1178724cfd2eSPaolo Abeni if (unlikely(tcp_under_memory_pressure(sk))) { 1179724cfd2eSPaolo Abeni mptcp_mem_reclaim_partial(sk); 1180724cfd2eSPaolo Abeni 1181724cfd2eSPaolo Abeni /* under pressure pre-allocate at most a single skb */ 1182724cfd2eSPaolo Abeni if (msk->skb_tx_cache.qlen) 1183724cfd2eSPaolo Abeni return true; 1184724cfd2eSPaolo Abeni space_needed = msk->size_goal_cache; 1185724cfd2eSPaolo Abeni } else { 1186724cfd2eSPaolo Abeni space_needed = msk->tx_pending_data + size - 1187724cfd2eSPaolo Abeni msk->skb_tx_cache.qlen * msk->size_goal_cache; 1188724cfd2eSPaolo Abeni } 1189724cfd2eSPaolo Abeni 1190724cfd2eSPaolo Abeni while (space_needed > 0) { 11916e628cd3SPaolo Abeni skb = __mptcp_do_alloc_tx_skb(sk, sk->sk_allocation); 1192724cfd2eSPaolo Abeni if (unlikely(!skb)) { 1193724cfd2eSPaolo Abeni /* under memory pressure, try to pass the caller a 1194724cfd2eSPaolo Abeni * single skb to allow forward progress 1195724cfd2eSPaolo Abeni */ 1196724cfd2eSPaolo Abeni while (skbs->qlen > 1) { 1197724cfd2eSPaolo Abeni skb = __skb_dequeue_tail(skbs); 1198eaeef1ceSPaolo Abeni *total_ts -= skb->truesize; 1199724cfd2eSPaolo Abeni __kfree_skb(skb); 1200724cfd2eSPaolo Abeni } 1201724cfd2eSPaolo Abeni return skbs->qlen > 0; 1202724cfd2eSPaolo Abeni } 1203724cfd2eSPaolo Abeni 1204724cfd2eSPaolo Abeni *total_ts += skb->truesize; 1205724cfd2eSPaolo Abeni __skb_queue_tail(skbs, skb); 1206724cfd2eSPaolo Abeni space_needed -= msk->size_goal_cache; 1207724cfd2eSPaolo Abeni } 1208724cfd2eSPaolo Abeni return true; 1209724cfd2eSPaolo Abeni } 1210724cfd2eSPaolo Abeni 12116e628cd3SPaolo Abeni static bool __mptcp_alloc_tx_skb(struct sock *sk, struct sock *ssk, gfp_t gfp) 1212724cfd2eSPaolo Abeni { 1213724cfd2eSPaolo Abeni struct mptcp_sock *msk = mptcp_sk(sk); 1214724cfd2eSPaolo Abeni struct sk_buff *skb; 1215724cfd2eSPaolo Abeni 1216724cfd2eSPaolo Abeni if (ssk->sk_tx_skb_cache) { 1217724cfd2eSPaolo Abeni skb = ssk->sk_tx_skb_cache; 1218724cfd2eSPaolo Abeni if (unlikely(!skb_ext_find(skb, SKB_EXT_MPTCP) && 12196e628cd3SPaolo Abeni !__mptcp_add_ext(skb, gfp))) 1220724cfd2eSPaolo Abeni return false; 1221724cfd2eSPaolo Abeni return true; 1222724cfd2eSPaolo Abeni } 1223724cfd2eSPaolo Abeni 1224724cfd2eSPaolo Abeni skb = skb_peek(&msk->skb_tx_cache); 1225724cfd2eSPaolo Abeni if (skb) { 1226724cfd2eSPaolo Abeni if (likely(sk_wmem_schedule(ssk, skb->truesize))) { 1227724cfd2eSPaolo Abeni skb = __skb_dequeue(&msk->skb_tx_cache); 1228724cfd2eSPaolo Abeni if (WARN_ON_ONCE(!skb)) 1229724cfd2eSPaolo Abeni return false; 1230724cfd2eSPaolo Abeni 1231724cfd2eSPaolo Abeni mptcp_wmem_uncharge(sk, skb->truesize); 1232724cfd2eSPaolo Abeni ssk->sk_tx_skb_cache = skb; 1233724cfd2eSPaolo Abeni return true; 1234724cfd2eSPaolo Abeni } 1235724cfd2eSPaolo Abeni 1236724cfd2eSPaolo Abeni /* over memory limit, no point to try to allocate a new skb */ 1237724cfd2eSPaolo Abeni return false; 1238724cfd2eSPaolo Abeni } 1239724cfd2eSPaolo Abeni 12406e628cd3SPaolo Abeni skb = __mptcp_do_alloc_tx_skb(sk, gfp); 1241724cfd2eSPaolo Abeni if (!skb) 1242724cfd2eSPaolo Abeni return false; 1243724cfd2eSPaolo Abeni 1244724cfd2eSPaolo Abeni if (likely(sk_wmem_schedule(ssk, skb->truesize))) { 1245724cfd2eSPaolo Abeni ssk->sk_tx_skb_cache = skb; 1246724cfd2eSPaolo Abeni return true; 1247724cfd2eSPaolo Abeni } 1248724cfd2eSPaolo Abeni kfree_skb(skb); 1249724cfd2eSPaolo Abeni return false; 1250724cfd2eSPaolo Abeni } 1251724cfd2eSPaolo Abeni 1252724cfd2eSPaolo Abeni static bool mptcp_must_reclaim_memory(struct sock *sk, struct sock *ssk) 1253724cfd2eSPaolo Abeni { 1254724cfd2eSPaolo Abeni return !ssk->sk_tx_skb_cache && 1255724cfd2eSPaolo Abeni !skb_peek(&mptcp_sk(sk)->skb_tx_cache) && 1256724cfd2eSPaolo Abeni tcp_under_memory_pressure(sk); 1257724cfd2eSPaolo Abeni } 1258724cfd2eSPaolo Abeni 1259724cfd2eSPaolo Abeni static bool mptcp_alloc_tx_skb(struct sock *sk, struct sock *ssk) 1260724cfd2eSPaolo Abeni { 1261724cfd2eSPaolo Abeni if (unlikely(mptcp_must_reclaim_memory(sk, ssk))) 1262724cfd2eSPaolo Abeni mptcp_mem_reclaim_partial(sk); 12636e628cd3SPaolo Abeni return __mptcp_alloc_tx_skb(sk, ssk, sk->sk_allocation); 1264724cfd2eSPaolo Abeni } 1265724cfd2eSPaolo Abeni 126657040755SPaolo Abeni static int mptcp_sendmsg_frag(struct sock *sk, struct sock *ssk, 1267d9ca1de8SPaolo Abeni struct mptcp_data_frag *dfrag, 1268caf971dfSPaolo Abeni struct mptcp_sendmsg_info *info) 126957040755SPaolo Abeni { 1270d9ca1de8SPaolo Abeni u64 data_seq = dfrag->data_seq + info->sent; 12716d0060f6SMat Martineau struct mptcp_sock *msk = mptcp_sk(sk); 12726f8a612aSFlorian Westphal bool zero_window_probe = false; 12736d0060f6SMat Martineau struct mptcp_ext *mpext = NULL; 127457040755SPaolo Abeni struct sk_buff *skb, *tail; 1275d9ca1de8SPaolo Abeni bool can_collapse = false; 127615e6ca97SPaolo Abeni int size_bias = 0; 1277d9ca1de8SPaolo Abeni int avail_size; 1278724cfd2eSPaolo Abeni size_t ret = 0; 12796d0060f6SMat Martineau 1280d9ca1de8SPaolo Abeni pr_debug("msk=%p ssk=%p sending dfrag at seq=%lld len=%d already sent=%d", 1281d9ca1de8SPaolo Abeni msk, ssk, dfrag->data_seq, dfrag->data_len, info->sent); 12826d0060f6SMat Martineau 1283d9ca1de8SPaolo Abeni /* compute send limit */ 1284d9ca1de8SPaolo Abeni info->mss_now = tcp_send_mss(ssk, &info->size_goal, info->flags); 1285caf971dfSPaolo Abeni avail_size = info->size_goal; 1286724cfd2eSPaolo Abeni msk->size_goal_cache = info->size_goal; 128757040755SPaolo Abeni skb = tcp_write_queue_tail(ssk); 128857040755SPaolo Abeni if (skb) { 128957040755SPaolo Abeni /* Limit the write to the size available in the 129057040755SPaolo Abeni * current skb, if any, so that we create at most a new skb. 129157040755SPaolo Abeni * Explicitly tells TCP internals to avoid collapsing on later 129257040755SPaolo Abeni * queue management operation, to avoid breaking the ext <-> 129357040755SPaolo Abeni * SSN association set here 129457040755SPaolo Abeni */ 1295d9ca1de8SPaolo Abeni mpext = skb_ext_find(skb, SKB_EXT_MPTCP); 1296caf971dfSPaolo Abeni can_collapse = (info->size_goal - skb->len > 0) && 1297d9ca1de8SPaolo Abeni mptcp_skb_can_collapse_to(data_seq, skb, mpext); 129815e6ca97SPaolo Abeni if (!can_collapse) { 129957040755SPaolo Abeni TCP_SKB_CB(skb)->eor = 1; 130015e6ca97SPaolo Abeni } else { 130115e6ca97SPaolo Abeni size_bias = skb->len; 1302caf971dfSPaolo Abeni avail_size = info->size_goal - skb->len; 130357040755SPaolo Abeni } 130415e6ca97SPaolo Abeni } 130518b683bfSPaolo Abeni 13066f8a612aSFlorian Westphal /* Zero window and all data acked? Probe. */ 13076f8a612aSFlorian Westphal avail_size = mptcp_check_allowed_size(msk, data_seq, avail_size); 13086f8a612aSFlorian Westphal if (avail_size == 0) { 13097439d687SPaolo Abeni u64 snd_una = READ_ONCE(msk->snd_una); 13107439d687SPaolo Abeni 13117439d687SPaolo Abeni if (skb || snd_una != msk->snd_nxt) 13126f8a612aSFlorian Westphal return 0; 13136f8a612aSFlorian Westphal zero_window_probe = true; 13147439d687SPaolo Abeni data_seq = snd_una - 1; 13156f8a612aSFlorian Westphal avail_size = 1; 13166f8a612aSFlorian Westphal } 13176f8a612aSFlorian Westphal 1318d9ca1de8SPaolo Abeni if (WARN_ON_ONCE(info->sent > info->limit || 1319d9ca1de8SPaolo Abeni info->limit > dfrag->data_len)) 1320d9ca1de8SPaolo Abeni return 0; 132157040755SPaolo Abeni 1322d9ca1de8SPaolo Abeni ret = info->limit - info->sent; 132315e6ca97SPaolo Abeni tail = tcp_build_frag(ssk, avail_size + size_bias, info->flags, 132415e6ca97SPaolo Abeni dfrag->page, dfrag->offset + info->sent, &ret); 1325e2223995SPaolo Abeni if (!tail) { 1326e2223995SPaolo Abeni tcp_remove_empty_skb(sk, tcp_write_queue_tail(ssk)); 1327e2223995SPaolo Abeni return -ENOMEM; 132835759383SFlorian Westphal } 132918b683bfSPaolo Abeni 1330e2223995SPaolo Abeni /* if the tail skb is still the cached one, collapsing really happened. 133157040755SPaolo Abeni */ 1332e2223995SPaolo Abeni if (skb == tail) { 133315e6ca97SPaolo Abeni TCP_SKB_CB(tail)->tcp_flags &= ~TCPHDR_PSH; 133457040755SPaolo Abeni mpext->data_len += ret; 133515e6ca97SPaolo Abeni WARN_ON_ONCE(!can_collapse); 13366f8a612aSFlorian Westphal WARN_ON_ONCE(zero_window_probe); 133757040755SPaolo Abeni goto out; 133857040755SPaolo Abeni } 133957040755SPaolo Abeni 1340724cfd2eSPaolo Abeni mpext = skb_ext_find(tail, SKB_EXT_MPTCP); 1341724cfd2eSPaolo Abeni if (WARN_ON_ONCE(!mpext)) { 1342724cfd2eSPaolo Abeni /* should never reach here, stream corrupted */ 1343724cfd2eSPaolo Abeni return -EINVAL; 1344724cfd2eSPaolo Abeni } 13456d0060f6SMat Martineau 13466d0060f6SMat Martineau memset(mpext, 0, sizeof(*mpext)); 1347d9ca1de8SPaolo Abeni mpext->data_seq = data_seq; 13486d0060f6SMat Martineau mpext->subflow_seq = mptcp_subflow_ctx(ssk)->rel_write_seq; 13496d0060f6SMat Martineau mpext->data_len = ret; 13506d0060f6SMat Martineau mpext->use_map = 1; 13516d0060f6SMat Martineau mpext->dsn64 = 1; 13526d0060f6SMat Martineau 13536d0060f6SMat Martineau pr_debug("data_seq=%llu subflow_seq=%u data_len=%u dsn64=%d", 13546d0060f6SMat Martineau mpext->data_seq, mpext->subflow_seq, mpext->data_len, 13556d0060f6SMat Martineau mpext->dsn64); 13566d0060f6SMat Martineau 13576f8a612aSFlorian Westphal if (zero_window_probe) { 13586f8a612aSFlorian Westphal mptcp_subflow_ctx(ssk)->rel_write_seq += ret; 13596f8a612aSFlorian Westphal mpext->frozen = 1; 13606f8a612aSFlorian Westphal ret = 0; 13616f8a612aSFlorian Westphal tcp_push_pending_frames(ssk); 13626f8a612aSFlorian Westphal } 136357040755SPaolo Abeni out: 13646d0060f6SMat Martineau mptcp_subflow_ctx(ssk)->rel_write_seq += ret; 13656d0060f6SMat Martineau return ret; 13666d0060f6SMat Martineau } 13676d0060f6SMat Martineau 1368d5f49190SPaolo Abeni #define MPTCP_SEND_BURST_SIZE ((1 << 16) - \ 1369d5f49190SPaolo Abeni sizeof(struct tcphdr) - \ 1370d5f49190SPaolo Abeni MAX_TCP_OPTION_SPACE - \ 1371d5f49190SPaolo Abeni sizeof(struct ipv6hdr) - \ 1372d5f49190SPaolo Abeni sizeof(struct frag_hdr)) 1373d5f49190SPaolo Abeni 1374d5f49190SPaolo Abeni struct subflow_send_info { 1375d5f49190SPaolo Abeni struct sock *ssk; 1376d5f49190SPaolo Abeni u64 ratio; 1377d5f49190SPaolo Abeni }; 1378d5f49190SPaolo Abeni 13795cf92bbaSPaolo Abeni static struct sock *mptcp_subflow_get_send(struct mptcp_sock *msk) 1380f296234cSPeter Krystad { 1381d5f49190SPaolo Abeni struct subflow_send_info send_info[2]; 1382f296234cSPeter Krystad struct mptcp_subflow_context *subflow; 1383d5f49190SPaolo Abeni int i, nr_active = 0; 1384d5f49190SPaolo Abeni struct sock *ssk; 1385d5f49190SPaolo Abeni u64 ratio; 1386d5f49190SPaolo Abeni u32 pace; 1387f296234cSPeter Krystad 1388d5f49190SPaolo Abeni sock_owned_by_me((struct sock *)msk); 1389f296234cSPeter Krystad 1390d5f49190SPaolo Abeni if (__mptcp_check_fallback(msk)) { 1391d5f49190SPaolo Abeni if (!msk->first) 1392f296234cSPeter Krystad return NULL; 1393d5f49190SPaolo Abeni return sk_stream_memory_free(msk->first) ? msk->first : NULL; 1394f296234cSPeter Krystad } 1395f296234cSPeter Krystad 1396d5f49190SPaolo Abeni /* re-use last subflow, if the burst allow that */ 1397d5f49190SPaolo Abeni if (msk->last_snd && msk->snd_burst > 0 && 1398d5f49190SPaolo Abeni sk_stream_memory_free(msk->last_snd) && 13995cf92bbaSPaolo Abeni mptcp_subflow_active(mptcp_subflow_ctx(msk->last_snd))) 1400d5f49190SPaolo Abeni return msk->last_snd; 1401f296234cSPeter Krystad 1402d5f49190SPaolo Abeni /* pick the subflow with the lower wmem/wspace ratio */ 1403d5f49190SPaolo Abeni for (i = 0; i < 2; ++i) { 1404d5f49190SPaolo Abeni send_info[i].ssk = NULL; 1405d5f49190SPaolo Abeni send_info[i].ratio = -1; 1406d5f49190SPaolo Abeni } 1407d5f49190SPaolo Abeni mptcp_for_each_subflow(msk, subflow) { 1408d5f49190SPaolo Abeni ssk = mptcp_subflow_tcp_sock(subflow); 1409d5f49190SPaolo Abeni if (!mptcp_subflow_active(subflow)) 1410f296234cSPeter Krystad continue; 1411d5f49190SPaolo Abeni 1412d5f49190SPaolo Abeni nr_active += !subflow->backup; 1413ec369c3aSPaolo Abeni if (!sk_stream_memory_free(subflow->tcp_sock) || !tcp_sk(ssk)->snd_wnd) 1414d5f49190SPaolo Abeni continue; 1415d5f49190SPaolo Abeni 1416d5f49190SPaolo Abeni pace = READ_ONCE(ssk->sk_pacing_rate); 1417d5f49190SPaolo Abeni if (!pace) 1418d5f49190SPaolo Abeni continue; 1419d5f49190SPaolo Abeni 1420d5f49190SPaolo Abeni ratio = div_u64((u64)READ_ONCE(ssk->sk_wmem_queued) << 32, 1421d5f49190SPaolo Abeni pace); 1422d5f49190SPaolo Abeni if (ratio < send_info[subflow->backup].ratio) { 1423d5f49190SPaolo Abeni send_info[subflow->backup].ssk = ssk; 1424d5f49190SPaolo Abeni send_info[subflow->backup].ratio = ratio; 1425d5f49190SPaolo Abeni } 1426f296234cSPeter Krystad } 1427f296234cSPeter Krystad 1428d5f49190SPaolo Abeni pr_debug("msk=%p nr_active=%d ssk=%p:%lld backup=%p:%lld", 1429d5f49190SPaolo Abeni msk, nr_active, send_info[0].ssk, send_info[0].ratio, 1430d5f49190SPaolo Abeni send_info[1].ssk, send_info[1].ratio); 1431f296234cSPeter Krystad 1432d5f49190SPaolo Abeni /* pick the best backup if no other subflow is active */ 1433d5f49190SPaolo Abeni if (!nr_active) 1434d5f49190SPaolo Abeni send_info[0].ssk = send_info[1].ssk; 1435d5f49190SPaolo Abeni 1436d5f49190SPaolo Abeni if (send_info[0].ssk) { 1437d5f49190SPaolo Abeni msk->last_snd = send_info[0].ssk; 1438d5f49190SPaolo Abeni msk->snd_burst = min_t(int, MPTCP_SEND_BURST_SIZE, 1439ec369c3aSPaolo Abeni tcp_sk(msk->last_snd)->snd_wnd); 1440d5f49190SPaolo Abeni return msk->last_snd; 1441d5f49190SPaolo Abeni } 14425cf92bbaSPaolo Abeni 1443d5f49190SPaolo Abeni return NULL; 1444f296234cSPeter Krystad } 1445f296234cSPeter Krystad 1446d9ca1de8SPaolo Abeni static void mptcp_push_release(struct sock *sk, struct sock *ssk, 1447d9ca1de8SPaolo Abeni struct mptcp_sendmsg_info *info) 1448d9ca1de8SPaolo Abeni { 1449d9ca1de8SPaolo Abeni mptcp_set_timeout(sk, ssk); 1450d9ca1de8SPaolo Abeni tcp_push(ssk, 0, info->mss_now, tcp_sk(ssk)->nonagle, info->size_goal); 1451d9ca1de8SPaolo Abeni release_sock(ssk); 1452d9ca1de8SPaolo Abeni } 1453d9ca1de8SPaolo Abeni 1454c2e6048fSPaolo Abeni static void __mptcp_push_pending(struct sock *sk, unsigned int flags) 1455d9ca1de8SPaolo Abeni { 1456d9ca1de8SPaolo Abeni struct sock *prev_ssk = NULL, *ssk = NULL; 1457d9ca1de8SPaolo Abeni struct mptcp_sock *msk = mptcp_sk(sk); 1458d9ca1de8SPaolo Abeni struct mptcp_sendmsg_info info = { 1459d9ca1de8SPaolo Abeni .flags = flags, 1460d9ca1de8SPaolo Abeni }; 1461d9ca1de8SPaolo Abeni struct mptcp_data_frag *dfrag; 1462d9ca1de8SPaolo Abeni int len, copied = 0; 1463d9ca1de8SPaolo Abeni 1464d9ca1de8SPaolo Abeni while ((dfrag = mptcp_send_head(sk))) { 1465d9ca1de8SPaolo Abeni info.sent = dfrag->already_sent; 1466d9ca1de8SPaolo Abeni info.limit = dfrag->data_len; 1467d9ca1de8SPaolo Abeni len = dfrag->data_len - dfrag->already_sent; 1468d9ca1de8SPaolo Abeni while (len > 0) { 1469d9ca1de8SPaolo Abeni int ret = 0; 1470d9ca1de8SPaolo Abeni 1471d9ca1de8SPaolo Abeni prev_ssk = ssk; 1472d9ca1de8SPaolo Abeni __mptcp_flush_join_list(msk); 14735cf92bbaSPaolo Abeni ssk = mptcp_subflow_get_send(msk); 1474d9ca1de8SPaolo Abeni 1475d9ca1de8SPaolo Abeni /* try to keep the subflow socket lock across 1476d9ca1de8SPaolo Abeni * consecutive xmit on the same socket 1477d9ca1de8SPaolo Abeni */ 1478d9ca1de8SPaolo Abeni if (ssk != prev_ssk && prev_ssk) 1479d9ca1de8SPaolo Abeni mptcp_push_release(sk, prev_ssk, &info); 1480d9ca1de8SPaolo Abeni if (!ssk) 1481d9ca1de8SPaolo Abeni goto out; 1482d9ca1de8SPaolo Abeni 1483d9ca1de8SPaolo Abeni if (ssk != prev_ssk || !prev_ssk) 1484d9ca1de8SPaolo Abeni lock_sock(ssk); 1485d9ca1de8SPaolo Abeni 1486724cfd2eSPaolo Abeni /* keep it simple and always provide a new skb for the 1487724cfd2eSPaolo Abeni * subflow, even if we will not use it when collapsing 1488724cfd2eSPaolo Abeni * on the pending one 1489724cfd2eSPaolo Abeni */ 1490724cfd2eSPaolo Abeni if (!mptcp_alloc_tx_skb(sk, ssk)) { 1491724cfd2eSPaolo Abeni mptcp_push_release(sk, ssk, &info); 1492724cfd2eSPaolo Abeni goto out; 1493724cfd2eSPaolo Abeni } 1494724cfd2eSPaolo Abeni 1495d9ca1de8SPaolo Abeni ret = mptcp_sendmsg_frag(sk, ssk, dfrag, &info); 1496d9ca1de8SPaolo Abeni if (ret <= 0) { 1497d9ca1de8SPaolo Abeni mptcp_push_release(sk, ssk, &info); 1498d9ca1de8SPaolo Abeni goto out; 1499d9ca1de8SPaolo Abeni } 1500d9ca1de8SPaolo Abeni 1501d9ca1de8SPaolo Abeni info.sent += ret; 1502d9ca1de8SPaolo Abeni dfrag->already_sent += ret; 1503d9ca1de8SPaolo Abeni msk->snd_nxt += ret; 1504d9ca1de8SPaolo Abeni msk->snd_burst -= ret; 1505724cfd2eSPaolo Abeni msk->tx_pending_data -= ret; 1506d9ca1de8SPaolo Abeni copied += ret; 1507d9ca1de8SPaolo Abeni len -= ret; 1508d9ca1de8SPaolo Abeni } 1509d9ca1de8SPaolo Abeni WRITE_ONCE(msk->first_pending, mptcp_send_next(sk)); 1510d9ca1de8SPaolo Abeni } 1511d9ca1de8SPaolo Abeni 1512d9ca1de8SPaolo Abeni /* at this point we held the socket lock for the last subflow we used */ 1513d9ca1de8SPaolo Abeni if (ssk) 1514d9ca1de8SPaolo Abeni mptcp_push_release(sk, ssk, &info); 1515d9ca1de8SPaolo Abeni 1516d9ca1de8SPaolo Abeni out: 1517b680a214SPaolo Abeni if (copied) { 1518d9ca1de8SPaolo Abeni /* start the timer, if it's not pending */ 1519d9ca1de8SPaolo Abeni if (!mptcp_timer_pending(sk)) 1520d9ca1de8SPaolo Abeni mptcp_reset_timer(sk); 1521d9ca1de8SPaolo Abeni __mptcp_check_send_data_fin(sk); 1522d9ca1de8SPaolo Abeni } 1523b680a214SPaolo Abeni } 1524d9ca1de8SPaolo Abeni 15256e628cd3SPaolo Abeni static void __mptcp_subflow_push_pending(struct sock *sk, struct sock *ssk) 15266e628cd3SPaolo Abeni { 15276e628cd3SPaolo Abeni struct mptcp_sock *msk = mptcp_sk(sk); 15286e628cd3SPaolo Abeni struct mptcp_sendmsg_info info; 15296e628cd3SPaolo Abeni struct mptcp_data_frag *dfrag; 1530b19bc294SPaolo Abeni struct sock *xmit_ssk; 15316e628cd3SPaolo Abeni int len, copied = 0; 1532b19bc294SPaolo Abeni bool first = true; 15336e628cd3SPaolo Abeni 15346e628cd3SPaolo Abeni info.flags = 0; 15356e628cd3SPaolo Abeni while ((dfrag = mptcp_send_head(sk))) { 15366e628cd3SPaolo Abeni info.sent = dfrag->already_sent; 15376e628cd3SPaolo Abeni info.limit = dfrag->data_len; 15386e628cd3SPaolo Abeni len = dfrag->data_len - dfrag->already_sent; 15396e628cd3SPaolo Abeni while (len > 0) { 15406e628cd3SPaolo Abeni int ret = 0; 15416e628cd3SPaolo Abeni 1542b19bc294SPaolo Abeni /* the caller already invoked the packet scheduler, 1543b19bc294SPaolo Abeni * check for a different subflow usage only after 1544b19bc294SPaolo Abeni * spooling the first chunk of data 1545b19bc294SPaolo Abeni */ 1546b19bc294SPaolo Abeni xmit_ssk = first ? ssk : mptcp_subflow_get_send(mptcp_sk(sk)); 1547b19bc294SPaolo Abeni if (!xmit_ssk) 1548b19bc294SPaolo Abeni goto out; 1549b19bc294SPaolo Abeni if (xmit_ssk != ssk) { 1550b19bc294SPaolo Abeni mptcp_subflow_delegate(mptcp_subflow_ctx(xmit_ssk)); 1551b19bc294SPaolo Abeni goto out; 1552b19bc294SPaolo Abeni } 1553b19bc294SPaolo Abeni 15546e628cd3SPaolo Abeni if (unlikely(mptcp_must_reclaim_memory(sk, ssk))) { 15556e628cd3SPaolo Abeni __mptcp_update_wmem(sk); 15566e628cd3SPaolo Abeni sk_mem_reclaim_partial(sk); 15576e628cd3SPaolo Abeni } 15586e628cd3SPaolo Abeni if (!__mptcp_alloc_tx_skb(sk, ssk, GFP_ATOMIC)) 15596e628cd3SPaolo Abeni goto out; 15606e628cd3SPaolo Abeni 15616e628cd3SPaolo Abeni ret = mptcp_sendmsg_frag(sk, ssk, dfrag, &info); 15626e628cd3SPaolo Abeni if (ret <= 0) 15636e628cd3SPaolo Abeni goto out; 15646e628cd3SPaolo Abeni 15656e628cd3SPaolo Abeni info.sent += ret; 15666e628cd3SPaolo Abeni dfrag->already_sent += ret; 15676e628cd3SPaolo Abeni msk->snd_nxt += ret; 15686e628cd3SPaolo Abeni msk->snd_burst -= ret; 15696e628cd3SPaolo Abeni msk->tx_pending_data -= ret; 15706e628cd3SPaolo Abeni copied += ret; 15716e628cd3SPaolo Abeni len -= ret; 1572b19bc294SPaolo Abeni first = false; 15736e628cd3SPaolo Abeni } 15746e628cd3SPaolo Abeni WRITE_ONCE(msk->first_pending, mptcp_send_next(sk)); 15756e628cd3SPaolo Abeni } 15766e628cd3SPaolo Abeni 15776e628cd3SPaolo Abeni out: 15786e628cd3SPaolo Abeni /* __mptcp_alloc_tx_skb could have released some wmem and we are 15796e628cd3SPaolo Abeni * not going to flush it via release_sock() 15806e628cd3SPaolo Abeni */ 15816e628cd3SPaolo Abeni __mptcp_update_wmem(sk); 15826e628cd3SPaolo Abeni if (copied) { 15836e628cd3SPaolo Abeni mptcp_set_timeout(sk, ssk); 15846e628cd3SPaolo Abeni tcp_push(ssk, 0, info.mss_now, tcp_sk(ssk)->nonagle, 15856e628cd3SPaolo Abeni info.size_goal); 1586d09d818eSPaolo Abeni if (!mptcp_timer_pending(sk)) 1587d09d818eSPaolo Abeni mptcp_reset_timer(sk); 1588d09d818eSPaolo Abeni 15896e628cd3SPaolo Abeni if (msk->snd_data_fin_enable && 15906e628cd3SPaolo Abeni msk->snd_nxt + 1 == msk->write_seq) 15916e628cd3SPaolo Abeni mptcp_schedule_work(sk); 15926e628cd3SPaolo Abeni } 15936e628cd3SPaolo Abeni } 15946e628cd3SPaolo Abeni 15955cf92bbaSPaolo Abeni static void mptcp_set_nospace(struct sock *sk) 15965cf92bbaSPaolo Abeni { 15975cf92bbaSPaolo Abeni /* enable autotune */ 15985cf92bbaSPaolo Abeni set_bit(SOCK_NOSPACE, &sk->sk_socket->flags); 15995cf92bbaSPaolo Abeni 16005cf92bbaSPaolo Abeni /* will be cleared on avail space */ 16015cf92bbaSPaolo Abeni set_bit(MPTCP_NOSPACE, &mptcp_sk(sk)->flags); 16025cf92bbaSPaolo Abeni } 16035cf92bbaSPaolo Abeni 1604f870fa0bSMat Martineau static int mptcp_sendmsg(struct sock *sk, struct msghdr *msg, size_t len) 1605f870fa0bSMat Martineau { 1606f870fa0bSMat Martineau struct mptcp_sock *msk = mptcp_sk(sk); 160717091708SFlorian Westphal struct page_frag *pfrag; 16086d0060f6SMat Martineau size_t copied = 0; 1609caf971dfSPaolo Abeni int ret = 0; 16106d0060f6SMat Martineau long timeo; 1611f870fa0bSMat Martineau 1612f870fa0bSMat Martineau if (msg->msg_flags & ~(MSG_MORE | MSG_DONTWAIT | MSG_NOSIGNAL)) 1613f870fa0bSMat Martineau return -EOPNOTSUPP; 1614f870fa0bSMat Martineau 1615e7579d5dSDavide Caratti mptcp_lock_sock(sk, __mptcp_wmem_reserve(sk, min_t(size_t, 1 << 20, len))); 16161954b860SMat Martineau 16171954b860SMat Martineau timeo = sock_sndtimeo(sk, msg->msg_flags & MSG_DONTWAIT); 16181954b860SMat Martineau 16191954b860SMat Martineau if ((1 << sk->sk_state) & ~(TCPF_ESTABLISHED | TCPF_CLOSE_WAIT)) { 16201954b860SMat Martineau ret = sk_stream_wait_connect(sk, &timeo); 16211954b860SMat Martineau if (ret) 16221954b860SMat Martineau goto out; 16231954b860SMat Martineau } 16241954b860SMat Martineau 162517091708SFlorian Westphal pfrag = sk_page_frag(sk); 162618b683bfSPaolo Abeni 1627d9ca1de8SPaolo Abeni while (msg_data_left(msg)) { 1628724cfd2eSPaolo Abeni int total_ts, frag_truesize = 0; 1629d9ca1de8SPaolo Abeni struct mptcp_data_frag *dfrag; 1630724cfd2eSPaolo Abeni struct sk_buff_head skbs; 1631d9ca1de8SPaolo Abeni bool dfrag_collapsed; 1632d9ca1de8SPaolo Abeni size_t psize, offset; 1633d9ca1de8SPaolo Abeni 163457baaf28SMat Martineau if (sk->sk_err || (sk->sk_shutdown & SEND_SHUTDOWN)) { 163557baaf28SMat Martineau ret = -EPIPE; 163657baaf28SMat Martineau goto out; 163757baaf28SMat Martineau } 163857baaf28SMat Martineau 1639d9ca1de8SPaolo Abeni /* reuse tail pfrag, if possible, or carve a new one from the 1640d9ca1de8SPaolo Abeni * page allocator 1641fb529e62SFlorian Westphal */ 1642d9ca1de8SPaolo Abeni dfrag = mptcp_pending_tail(sk); 1643d9ca1de8SPaolo Abeni dfrag_collapsed = mptcp_frag_can_collapse_to(msk, pfrag, dfrag); 1644d9ca1de8SPaolo Abeni if (!dfrag_collapsed) { 1645d9ca1de8SPaolo Abeni if (!sk_stream_memory_free(sk)) 1646d9ca1de8SPaolo Abeni goto wait_for_memory; 16476e628cd3SPaolo Abeni 1648d9ca1de8SPaolo Abeni if (!mptcp_page_frag_refill(sk, pfrag)) 1649d9ca1de8SPaolo Abeni goto wait_for_memory; 1650d9ca1de8SPaolo Abeni 1651d9ca1de8SPaolo Abeni dfrag = mptcp_carve_data_frag(msk, pfrag, pfrag->offset); 1652d9ca1de8SPaolo Abeni frag_truesize = dfrag->overhead; 1653fb529e62SFlorian Westphal } 1654fb529e62SFlorian Westphal 1655d9ca1de8SPaolo Abeni /* we do not bound vs wspace, to allow a single packet. 1656d9ca1de8SPaolo Abeni * memory accounting will prevent execessive memory usage 1657d9ca1de8SPaolo Abeni * anyway 1658d9ca1de8SPaolo Abeni */ 1659d9ca1de8SPaolo Abeni offset = dfrag->offset + dfrag->data_len; 1660d9ca1de8SPaolo Abeni psize = pfrag->size - offset; 1661d9ca1de8SPaolo Abeni psize = min_t(size_t, psize, msg_data_left(msg)); 1662724cfd2eSPaolo Abeni total_ts = psize + frag_truesize; 1663724cfd2eSPaolo Abeni __skb_queue_head_init(&skbs); 1664724cfd2eSPaolo Abeni if (!mptcp_tx_cache_refill(sk, psize, &skbs, &total_ts)) 1665d9ca1de8SPaolo Abeni goto wait_for_memory; 1666d9ca1de8SPaolo Abeni 1667724cfd2eSPaolo Abeni if (!mptcp_wmem_alloc(sk, total_ts)) { 1668724cfd2eSPaolo Abeni __skb_queue_purge(&skbs); 1669724cfd2eSPaolo Abeni goto wait_for_memory; 1670724cfd2eSPaolo Abeni } 1671724cfd2eSPaolo Abeni 1672724cfd2eSPaolo Abeni skb_queue_splice_tail(&skbs, &msk->skb_tx_cache); 1673d9ca1de8SPaolo Abeni if (copy_page_from_iter(dfrag->page, offset, psize, 1674d9ca1de8SPaolo Abeni &msg->msg_iter) != psize) { 167587952603SPaolo Abeni mptcp_wmem_uncharge(sk, psize + frag_truesize); 1676d9ca1de8SPaolo Abeni ret = -EFAULT; 1677d9ca1de8SPaolo Abeni goto out; 1678d9ca1de8SPaolo Abeni } 1679d9ca1de8SPaolo Abeni 1680d9ca1de8SPaolo Abeni /* data successfully copied into the write queue */ 1681d9ca1de8SPaolo Abeni copied += psize; 1682d9ca1de8SPaolo Abeni dfrag->data_len += psize; 1683d9ca1de8SPaolo Abeni frag_truesize += psize; 1684d9ca1de8SPaolo Abeni pfrag->offset += frag_truesize; 1685d9ca1de8SPaolo Abeni WRITE_ONCE(msk->write_seq, msk->write_seq + psize); 168613e16037SPaolo Abeni msk->tx_pending_data += psize; 1687d9ca1de8SPaolo Abeni 1688d9ca1de8SPaolo Abeni /* charge data on mptcp pending queue to the msk socket 1689d9ca1de8SPaolo Abeni * Note: we charge such data both to sk and ssk 1690d9ca1de8SPaolo Abeni */ 1691d9ca1de8SPaolo Abeni sk_wmem_queued_add(sk, frag_truesize); 1692d9ca1de8SPaolo Abeni if (!dfrag_collapsed) { 1693d9ca1de8SPaolo Abeni get_page(dfrag->page); 1694d9ca1de8SPaolo Abeni list_add_tail(&dfrag->list, &msk->rtx_queue); 1695d9ca1de8SPaolo Abeni if (!msk->first_pending) 1696d9ca1de8SPaolo Abeni WRITE_ONCE(msk->first_pending, dfrag); 1697d9ca1de8SPaolo Abeni } 1698d9ca1de8SPaolo Abeni pr_debug("msk=%p dfrag at seq=%lld len=%d sent=%d new=%d", msk, 1699d9ca1de8SPaolo Abeni dfrag->data_seq, dfrag->data_len, dfrag->already_sent, 1700d9ca1de8SPaolo Abeni !dfrag_collapsed); 1701d9ca1de8SPaolo Abeni 1702d9ca1de8SPaolo Abeni continue; 1703d9ca1de8SPaolo Abeni 1704d9ca1de8SPaolo Abeni wait_for_memory: 17055cf92bbaSPaolo Abeni mptcp_set_nospace(sk); 1706c2e6048fSPaolo Abeni __mptcp_push_pending(sk, msg->msg_flags); 1707f296234cSPeter Krystad ret = sk_stream_wait_memory(sk, &timeo); 1708f296234cSPeter Krystad if (ret) 1709f296234cSPeter Krystad goto out; 1710cec37a6eSPeter Krystad } 1711cec37a6eSPeter Krystad 171213e16037SPaolo Abeni if (copied) 1713c2e6048fSPaolo Abeni __mptcp_push_pending(sk, msg->msg_flags); 1714da51aef5SPaolo Abeni 17151954b860SMat Martineau out: 1716cec37a6eSPeter Krystad release_sock(sk); 17178555c6bfSPaolo Abeni return copied ? : ret; 1718f870fa0bSMat Martineau } 1719f870fa0bSMat Martineau 17207a6a6cbcSPaolo Abeni static void mptcp_wait_data(struct sock *sk, long *timeo) 17217a6a6cbcSPaolo Abeni { 17227a6a6cbcSPaolo Abeni DEFINE_WAIT_FUNC(wait, woken_wake_function); 17237a6a6cbcSPaolo Abeni struct mptcp_sock *msk = mptcp_sk(sk); 17247a6a6cbcSPaolo Abeni 17257a6a6cbcSPaolo Abeni add_wait_queue(sk_sleep(sk), &wait); 17267a6a6cbcSPaolo Abeni sk_set_bit(SOCKWQ_ASYNC_WAITDATA, sk); 17277a6a6cbcSPaolo Abeni 17287a6a6cbcSPaolo Abeni sk_wait_event(sk, timeo, 17297a6a6cbcSPaolo Abeni test_and_clear_bit(MPTCP_DATA_READY, &msk->flags), &wait); 17307a6a6cbcSPaolo Abeni 17317a6a6cbcSPaolo Abeni sk_clear_bit(SOCKWQ_ASYNC_WAITDATA, sk); 17327a6a6cbcSPaolo Abeni remove_wait_queue(sk_sleep(sk), &wait); 17337a6a6cbcSPaolo Abeni } 17347a6a6cbcSPaolo Abeni 17356771bfd9SFlorian Westphal static int __mptcp_recvmsg_mskq(struct mptcp_sock *msk, 17366771bfd9SFlorian Westphal struct msghdr *msg, 17376771bfd9SFlorian Westphal size_t len) 17386771bfd9SFlorian Westphal { 17396771bfd9SFlorian Westphal struct sk_buff *skb; 17406771bfd9SFlorian Westphal int copied = 0; 17416771bfd9SFlorian Westphal 174287952603SPaolo Abeni while ((skb = skb_peek(&msk->receive_queue)) != NULL) { 17436771bfd9SFlorian Westphal u32 offset = MPTCP_SKB_CB(skb)->offset; 17446771bfd9SFlorian Westphal u32 data_len = skb->len - offset; 17456771bfd9SFlorian Westphal u32 count = min_t(size_t, len - copied, data_len); 17466771bfd9SFlorian Westphal int err; 17476771bfd9SFlorian Westphal 17486771bfd9SFlorian Westphal err = skb_copy_datagram_msg(skb, offset, msg, count); 17496771bfd9SFlorian Westphal if (unlikely(err < 0)) { 17506771bfd9SFlorian Westphal if (!copied) 17516771bfd9SFlorian Westphal return err; 17526771bfd9SFlorian Westphal break; 17536771bfd9SFlorian Westphal } 17546771bfd9SFlorian Westphal 17556771bfd9SFlorian Westphal copied += count; 17566771bfd9SFlorian Westphal 17576771bfd9SFlorian Westphal if (count < data_len) { 17586771bfd9SFlorian Westphal MPTCP_SKB_CB(skb)->offset += count; 17596771bfd9SFlorian Westphal break; 17606771bfd9SFlorian Westphal } 17616771bfd9SFlorian Westphal 176287952603SPaolo Abeni /* we will bulk release the skb memory later */ 176387952603SPaolo Abeni skb->destructor = NULL; 176487952603SPaolo Abeni msk->rmem_released += skb->truesize; 176587952603SPaolo Abeni __skb_unlink(skb, &msk->receive_queue); 17666771bfd9SFlorian Westphal __kfree_skb(skb); 17676771bfd9SFlorian Westphal 17686771bfd9SFlorian Westphal if (copied >= len) 17696771bfd9SFlorian Westphal break; 17706771bfd9SFlorian Westphal } 17716771bfd9SFlorian Westphal 17726771bfd9SFlorian Westphal return copied; 17736771bfd9SFlorian Westphal } 17746771bfd9SFlorian Westphal 1775a6b118feSFlorian Westphal /* receive buffer autotuning. See tcp_rcv_space_adjust for more information. 1776a6b118feSFlorian Westphal * 1777a6b118feSFlorian Westphal * Only difference: Use highest rtt estimate of the subflows in use. 1778a6b118feSFlorian Westphal */ 1779a6b118feSFlorian Westphal static void mptcp_rcv_space_adjust(struct mptcp_sock *msk, int copied) 1780a6b118feSFlorian Westphal { 1781a6b118feSFlorian Westphal struct mptcp_subflow_context *subflow; 1782a6b118feSFlorian Westphal struct sock *sk = (struct sock *)msk; 1783a6b118feSFlorian Westphal u32 time, advmss = 1; 1784a6b118feSFlorian Westphal u64 rtt_us, mstamp; 1785a6b118feSFlorian Westphal 1786a6b118feSFlorian Westphal sock_owned_by_me(sk); 1787a6b118feSFlorian Westphal 1788a6b118feSFlorian Westphal if (copied <= 0) 1789a6b118feSFlorian Westphal return; 1790a6b118feSFlorian Westphal 1791a6b118feSFlorian Westphal msk->rcvq_space.copied += copied; 1792a6b118feSFlorian Westphal 1793a6b118feSFlorian Westphal mstamp = div_u64(tcp_clock_ns(), NSEC_PER_USEC); 1794a6b118feSFlorian Westphal time = tcp_stamp_us_delta(mstamp, msk->rcvq_space.time); 1795a6b118feSFlorian Westphal 1796a6b118feSFlorian Westphal rtt_us = msk->rcvq_space.rtt_us; 1797a6b118feSFlorian Westphal if (rtt_us && time < (rtt_us >> 3)) 1798a6b118feSFlorian Westphal return; 1799a6b118feSFlorian Westphal 1800a6b118feSFlorian Westphal rtt_us = 0; 1801a6b118feSFlorian Westphal mptcp_for_each_subflow(msk, subflow) { 1802a6b118feSFlorian Westphal const struct tcp_sock *tp; 1803a6b118feSFlorian Westphal u64 sf_rtt_us; 1804a6b118feSFlorian Westphal u32 sf_advmss; 1805a6b118feSFlorian Westphal 1806a6b118feSFlorian Westphal tp = tcp_sk(mptcp_subflow_tcp_sock(subflow)); 1807a6b118feSFlorian Westphal 1808a6b118feSFlorian Westphal sf_rtt_us = READ_ONCE(tp->rcv_rtt_est.rtt_us); 1809a6b118feSFlorian Westphal sf_advmss = READ_ONCE(tp->advmss); 1810a6b118feSFlorian Westphal 1811a6b118feSFlorian Westphal rtt_us = max(sf_rtt_us, rtt_us); 1812a6b118feSFlorian Westphal advmss = max(sf_advmss, advmss); 1813a6b118feSFlorian Westphal } 1814a6b118feSFlorian Westphal 1815a6b118feSFlorian Westphal msk->rcvq_space.rtt_us = rtt_us; 1816a6b118feSFlorian Westphal if (time < (rtt_us >> 3) || rtt_us == 0) 1817a6b118feSFlorian Westphal return; 1818a6b118feSFlorian Westphal 1819a6b118feSFlorian Westphal if (msk->rcvq_space.copied <= msk->rcvq_space.space) 1820a6b118feSFlorian Westphal goto new_measure; 1821a6b118feSFlorian Westphal 1822a6b118feSFlorian Westphal if (sock_net(sk)->ipv4.sysctl_tcp_moderate_rcvbuf && 1823a6b118feSFlorian Westphal !(sk->sk_userlocks & SOCK_RCVBUF_LOCK)) { 1824a6b118feSFlorian Westphal int rcvmem, rcvbuf; 1825a6b118feSFlorian Westphal u64 rcvwin, grow; 1826a6b118feSFlorian Westphal 1827a6b118feSFlorian Westphal rcvwin = ((u64)msk->rcvq_space.copied << 1) + 16 * advmss; 1828a6b118feSFlorian Westphal 1829a6b118feSFlorian Westphal grow = rcvwin * (msk->rcvq_space.copied - msk->rcvq_space.space); 1830a6b118feSFlorian Westphal 1831a6b118feSFlorian Westphal do_div(grow, msk->rcvq_space.space); 1832a6b118feSFlorian Westphal rcvwin += (grow << 1); 1833a6b118feSFlorian Westphal 1834a6b118feSFlorian Westphal rcvmem = SKB_TRUESIZE(advmss + MAX_TCP_HEADER); 1835a6b118feSFlorian Westphal while (tcp_win_from_space(sk, rcvmem) < advmss) 1836a6b118feSFlorian Westphal rcvmem += 128; 1837a6b118feSFlorian Westphal 1838a6b118feSFlorian Westphal do_div(rcvwin, advmss); 1839a6b118feSFlorian Westphal rcvbuf = min_t(u64, rcvwin * rcvmem, 1840a6b118feSFlorian Westphal sock_net(sk)->ipv4.sysctl_tcp_rmem[2]); 1841a6b118feSFlorian Westphal 1842a6b118feSFlorian Westphal if (rcvbuf > sk->sk_rcvbuf) { 1843a6b118feSFlorian Westphal u32 window_clamp; 1844a6b118feSFlorian Westphal 1845a6b118feSFlorian Westphal window_clamp = tcp_win_from_space(sk, rcvbuf); 1846a6b118feSFlorian Westphal WRITE_ONCE(sk->sk_rcvbuf, rcvbuf); 1847a6b118feSFlorian Westphal 1848a6b118feSFlorian Westphal /* Make subflows follow along. If we do not do this, we 1849a6b118feSFlorian Westphal * get drops at subflow level if skbs can't be moved to 1850a6b118feSFlorian Westphal * the mptcp rx queue fast enough (announced rcv_win can 1851a6b118feSFlorian Westphal * exceed ssk->sk_rcvbuf). 1852a6b118feSFlorian Westphal */ 1853a6b118feSFlorian Westphal mptcp_for_each_subflow(msk, subflow) { 1854a6b118feSFlorian Westphal struct sock *ssk; 1855c76c6956SPaolo Abeni bool slow; 1856a6b118feSFlorian Westphal 1857a6b118feSFlorian Westphal ssk = mptcp_subflow_tcp_sock(subflow); 1858c76c6956SPaolo Abeni slow = lock_sock_fast(ssk); 1859a6b118feSFlorian Westphal WRITE_ONCE(ssk->sk_rcvbuf, rcvbuf); 1860a6b118feSFlorian Westphal tcp_sk(ssk)->window_clamp = window_clamp; 1861c76c6956SPaolo Abeni tcp_cleanup_rbuf(ssk, 1); 1862c76c6956SPaolo Abeni unlock_sock_fast(ssk, slow); 1863a6b118feSFlorian Westphal } 1864a6b118feSFlorian Westphal } 1865a6b118feSFlorian Westphal } 1866a6b118feSFlorian Westphal 1867a6b118feSFlorian Westphal msk->rcvq_space.space = msk->rcvq_space.copied; 1868a6b118feSFlorian Westphal new_measure: 1869a6b118feSFlorian Westphal msk->rcvq_space.copied = 0; 1870a6b118feSFlorian Westphal msk->rcvq_space.time = mstamp; 1871a6b118feSFlorian Westphal } 1872a6b118feSFlorian Westphal 187387952603SPaolo Abeni static void __mptcp_update_rmem(struct sock *sk) 187487952603SPaolo Abeni { 187587952603SPaolo Abeni struct mptcp_sock *msk = mptcp_sk(sk); 187687952603SPaolo Abeni 187787952603SPaolo Abeni if (!msk->rmem_released) 187887952603SPaolo Abeni return; 187987952603SPaolo Abeni 188087952603SPaolo Abeni atomic_sub(msk->rmem_released, &sk->sk_rmem_alloc); 188187952603SPaolo Abeni sk_mem_uncharge(sk, msk->rmem_released); 188287952603SPaolo Abeni msk->rmem_released = 0; 188387952603SPaolo Abeni } 188487952603SPaolo Abeni 188587952603SPaolo Abeni static void __mptcp_splice_receive_queue(struct sock *sk) 188687952603SPaolo Abeni { 188787952603SPaolo Abeni struct mptcp_sock *msk = mptcp_sk(sk); 188887952603SPaolo Abeni 188987952603SPaolo Abeni skb_queue_splice_tail_init(&sk->sk_receive_queue, &msk->receive_queue); 189087952603SPaolo Abeni } 189187952603SPaolo Abeni 1892e3859603SPaolo Abeni static bool __mptcp_move_skbs(struct mptcp_sock *msk) 18936771bfd9SFlorian Westphal { 189487952603SPaolo Abeni struct sock *sk = (struct sock *)msk; 18956771bfd9SFlorian Westphal unsigned int moved = 0; 189687952603SPaolo Abeni bool ret, done; 1897d5f49190SPaolo Abeni 1898d5f49190SPaolo Abeni __mptcp_flush_join_list(msk); 18996771bfd9SFlorian Westphal do { 19006771bfd9SFlorian Westphal struct sock *ssk = mptcp_subflow_recv_lookup(msk); 190165f49fe7SFlorian Westphal bool slowpath; 19026771bfd9SFlorian Westphal 190387952603SPaolo Abeni /* we can have data pending in the subflows only if the msk 190487952603SPaolo Abeni * receive buffer was full at subflow_data_ready() time, 190587952603SPaolo Abeni * that is an unlikely slow path. 190687952603SPaolo Abeni */ 190787952603SPaolo Abeni if (likely(!ssk)) 19086771bfd9SFlorian Westphal break; 19096771bfd9SFlorian Westphal 191065f49fe7SFlorian Westphal slowpath = lock_sock_fast(ssk); 191187952603SPaolo Abeni mptcp_data_lock(sk); 1912e3859603SPaolo Abeni __mptcp_update_rmem(sk); 19136771bfd9SFlorian Westphal done = __mptcp_move_skbs_from_subflow(msk, ssk, &moved); 191487952603SPaolo Abeni mptcp_data_unlock(sk); 1915e3859603SPaolo Abeni tcp_cleanup_rbuf(ssk, moved); 191665f49fe7SFlorian Westphal unlock_sock_fast(ssk, slowpath); 19176771bfd9SFlorian Westphal } while (!done); 19186771bfd9SFlorian Westphal 191987952603SPaolo Abeni /* acquire the data lock only if some input data is pending */ 192087952603SPaolo Abeni ret = moved > 0; 192187952603SPaolo Abeni if (!RB_EMPTY_ROOT(&msk->out_of_order_queue) || 192287952603SPaolo Abeni !skb_queue_empty_lockless(&sk->sk_receive_queue)) { 192387952603SPaolo Abeni mptcp_data_lock(sk); 192487952603SPaolo Abeni __mptcp_update_rmem(sk); 192587952603SPaolo Abeni ret |= __mptcp_ofo_queue(msk); 192687952603SPaolo Abeni __mptcp_splice_receive_queue(sk); 192787952603SPaolo Abeni mptcp_data_unlock(sk); 1928e3859603SPaolo Abeni mptcp_cleanup_rbuf(msk); 1929ab174ad8SPaolo Abeni } 193087952603SPaolo Abeni if (ret) 193187952603SPaolo Abeni mptcp_check_data_fin((struct sock *)msk); 193287952603SPaolo Abeni return !skb_queue_empty(&msk->receive_queue); 19336771bfd9SFlorian Westphal } 19346771bfd9SFlorian Westphal 1935f870fa0bSMat Martineau static int mptcp_recvmsg(struct sock *sk, struct msghdr *msg, size_t len, 1936f870fa0bSMat Martineau int nonblock, int flags, int *addr_len) 1937f870fa0bSMat Martineau { 1938f870fa0bSMat Martineau struct mptcp_sock *msk = mptcp_sk(sk); 1939cec37a6eSPeter Krystad int copied = 0; 19407a6a6cbcSPaolo Abeni int target; 19417a6a6cbcSPaolo Abeni long timeo; 1942f870fa0bSMat Martineau 1943f870fa0bSMat Martineau if (msg->msg_flags & ~(MSG_WAITALL | MSG_DONTWAIT)) 1944f870fa0bSMat Martineau return -EOPNOTSUPP; 1945f870fa0bSMat Martineau 194687952603SPaolo Abeni mptcp_lock_sock(sk, __mptcp_splice_receive_queue(sk)); 1947fd897679SPaolo Abeni if (unlikely(sk->sk_state == TCP_LISTEN)) { 1948fd897679SPaolo Abeni copied = -ENOTCONN; 1949fd897679SPaolo Abeni goto out_err; 1950fd897679SPaolo Abeni } 1951fd897679SPaolo Abeni 19527a6a6cbcSPaolo Abeni timeo = sock_rcvtimeo(sk, nonblock); 19537a6a6cbcSPaolo Abeni 19547a6a6cbcSPaolo Abeni len = min_t(size_t, len, INT_MAX); 19557a6a6cbcSPaolo Abeni target = sock_rcvlowat(sk, flags & MSG_WAITALL, len); 19567a6a6cbcSPaolo Abeni 195705e3eceaSEric Dumazet while (copied < len) { 1958e3859603SPaolo Abeni int bytes_read; 19597a6a6cbcSPaolo Abeni 19606771bfd9SFlorian Westphal bytes_read = __mptcp_recvmsg_mskq(msk, msg, len - copied); 19616771bfd9SFlorian Westphal if (unlikely(bytes_read < 0)) { 19627a6a6cbcSPaolo Abeni if (!copied) 19637a6a6cbcSPaolo Abeni copied = bytes_read; 19646771bfd9SFlorian Westphal goto out_err; 1965cec37a6eSPeter Krystad } 1966cec37a6eSPeter Krystad 19677a6a6cbcSPaolo Abeni copied += bytes_read; 19687a6a6cbcSPaolo Abeni 1969ea4ca586SPaolo Abeni /* be sure to advertise window change */ 1970fd897679SPaolo Abeni mptcp_cleanup_rbuf(msk); 1971ea4ca586SPaolo Abeni 1972e3859603SPaolo Abeni if (skb_queue_empty(&msk->receive_queue) && __mptcp_move_skbs(msk)) 1973e3859603SPaolo Abeni continue; 1974e3859603SPaolo Abeni 19757a6a6cbcSPaolo Abeni /* only the master socket status is relevant here. The exit 19767a6a6cbcSPaolo Abeni * conditions mirror closely tcp_recvmsg() 19777a6a6cbcSPaolo Abeni */ 19787a6a6cbcSPaolo Abeni if (copied >= target) 19797a6a6cbcSPaolo Abeni break; 19807a6a6cbcSPaolo Abeni 19817a6a6cbcSPaolo Abeni if (copied) { 19827a6a6cbcSPaolo Abeni if (sk->sk_err || 19837a6a6cbcSPaolo Abeni sk->sk_state == TCP_CLOSE || 19847a6a6cbcSPaolo Abeni (sk->sk_shutdown & RCV_SHUTDOWN) || 19857a6a6cbcSPaolo Abeni !timeo || 19867a6a6cbcSPaolo Abeni signal_pending(current)) 19877a6a6cbcSPaolo Abeni break; 19887a6a6cbcSPaolo Abeni } else { 19897a6a6cbcSPaolo Abeni if (sk->sk_err) { 19907a6a6cbcSPaolo Abeni copied = sock_error(sk); 19917a6a6cbcSPaolo Abeni break; 19927a6a6cbcSPaolo Abeni } 19937a6a6cbcSPaolo Abeni 19945969856aSPaolo Abeni if (test_and_clear_bit(MPTCP_WORK_EOF, &msk->flags)) 19955969856aSPaolo Abeni mptcp_check_for_eof(msk); 19965969856aSPaolo Abeni 199787952603SPaolo Abeni if (sk->sk_shutdown & RCV_SHUTDOWN) { 199887952603SPaolo Abeni /* race breaker: the shutdown could be after the 199987952603SPaolo Abeni * previous receive queue check 200087952603SPaolo Abeni */ 2001e3859603SPaolo Abeni if (__mptcp_move_skbs(msk)) 200287952603SPaolo Abeni continue; 20037a6a6cbcSPaolo Abeni break; 200487952603SPaolo Abeni } 20057a6a6cbcSPaolo Abeni 20067a6a6cbcSPaolo Abeni if (sk->sk_state == TCP_CLOSE) { 20077a6a6cbcSPaolo Abeni copied = -ENOTCONN; 20087a6a6cbcSPaolo Abeni break; 20097a6a6cbcSPaolo Abeni } 20107a6a6cbcSPaolo Abeni 20117a6a6cbcSPaolo Abeni if (!timeo) { 20127a6a6cbcSPaolo Abeni copied = -EAGAIN; 20137a6a6cbcSPaolo Abeni break; 20147a6a6cbcSPaolo Abeni } 20157a6a6cbcSPaolo Abeni 20167a6a6cbcSPaolo Abeni if (signal_pending(current)) { 20177a6a6cbcSPaolo Abeni copied = sock_intr_errno(timeo); 20187a6a6cbcSPaolo Abeni break; 20197a6a6cbcSPaolo Abeni } 20207a6a6cbcSPaolo Abeni } 20217a6a6cbcSPaolo Abeni 20227a6a6cbcSPaolo Abeni pr_debug("block timeout %ld", timeo); 20237a6a6cbcSPaolo Abeni mptcp_wait_data(sk, &timeo); 20247a6a6cbcSPaolo Abeni } 20257a6a6cbcSPaolo Abeni 202687952603SPaolo Abeni if (skb_queue_empty_lockless(&sk->sk_receive_queue) && 202787952603SPaolo Abeni skb_queue_empty(&msk->receive_queue)) { 20286771bfd9SFlorian Westphal /* entire backlog drained, clear DATA_READY. */ 20297a6a6cbcSPaolo Abeni clear_bit(MPTCP_DATA_READY, &msk->flags); 20307a6a6cbcSPaolo Abeni 20316771bfd9SFlorian Westphal /* .. race-breaker: ssk might have gotten new data 20326771bfd9SFlorian Westphal * after last __mptcp_move_skbs() returned false. 20337a6a6cbcSPaolo Abeni */ 2034e3859603SPaolo Abeni if (unlikely(__mptcp_move_skbs(msk))) 20356771bfd9SFlorian Westphal set_bit(MPTCP_DATA_READY, &msk->flags); 20366771bfd9SFlorian Westphal } else if (unlikely(!test_bit(MPTCP_DATA_READY, &msk->flags))) { 20376771bfd9SFlorian Westphal /* data to read but mptcp_wait_data() cleared DATA_READY */ 20387a6a6cbcSPaolo Abeni set_bit(MPTCP_DATA_READY, &msk->flags); 20397a6a6cbcSPaolo Abeni } 20406771bfd9SFlorian Westphal out_err: 20416719331cSPaolo Abeni pr_debug("msk=%p data_ready=%d rx queue empty=%d copied=%d", 20426719331cSPaolo Abeni msk, test_bit(MPTCP_DATA_READY, &msk->flags), 204387952603SPaolo Abeni skb_queue_empty_lockless(&sk->sk_receive_queue), copied); 2044a6b118feSFlorian Westphal mptcp_rcv_space_adjust(msk, copied); 2045a6b118feSFlorian Westphal 2046cec37a6eSPeter Krystad release_sock(sk); 2047cec37a6eSPeter Krystad return copied; 2048cec37a6eSPeter Krystad } 2049cec37a6eSPeter Krystad 2050b51f9b80SPaolo Abeni static void mptcp_retransmit_timer(struct timer_list *t) 2051b51f9b80SPaolo Abeni { 2052b51f9b80SPaolo Abeni struct inet_connection_sock *icsk = from_timer(icsk, t, 2053b51f9b80SPaolo Abeni icsk_retransmit_timer); 2054b51f9b80SPaolo Abeni struct sock *sk = &icsk->icsk_inet.sk; 20552d6f5a2bSPaolo Abeni struct mptcp_sock *msk = mptcp_sk(sk); 2056b51f9b80SPaolo Abeni 2057b51f9b80SPaolo Abeni bh_lock_sock(sk); 2058b51f9b80SPaolo Abeni if (!sock_owned_by_user(sk)) { 20592d6f5a2bSPaolo Abeni /* we need a process context to retransmit */ 20602d6f5a2bSPaolo Abeni if (!test_and_set_bit(MPTCP_WORK_RTX, &msk->flags)) 20612d6f5a2bSPaolo Abeni mptcp_schedule_work(sk); 2062b51f9b80SPaolo Abeni } else { 2063b51f9b80SPaolo Abeni /* delegate our work to tcp_release_cb() */ 20642d6f5a2bSPaolo Abeni set_bit(MPTCP_RETRANSMIT, &msk->flags); 2065b51f9b80SPaolo Abeni } 2066b51f9b80SPaolo Abeni bh_unlock_sock(sk); 2067b51f9b80SPaolo Abeni sock_put(sk); 2068b51f9b80SPaolo Abeni } 2069b51f9b80SPaolo Abeni 2070e16163b6SPaolo Abeni static void mptcp_timeout_timer(struct timer_list *t) 2071e16163b6SPaolo Abeni { 2072e16163b6SPaolo Abeni struct sock *sk = from_timer(sk, t, sk_timer); 2073e16163b6SPaolo Abeni 2074e16163b6SPaolo Abeni mptcp_schedule_work(sk); 2075b6d69fc8SFlorian Westphal sock_put(sk); 2076e16163b6SPaolo Abeni } 2077e16163b6SPaolo Abeni 20783b1d6210SPaolo Abeni /* Find an idle subflow. Return NULL if there is unacked data at tcp 20793b1d6210SPaolo Abeni * level. 20803b1d6210SPaolo Abeni * 20813b1d6210SPaolo Abeni * A backup subflow is returned only if that is the only kind available. 20823b1d6210SPaolo Abeni */ 20833b1d6210SPaolo Abeni static struct sock *mptcp_subflow_get_retrans(const struct mptcp_sock *msk) 20843b1d6210SPaolo Abeni { 20853b1d6210SPaolo Abeni struct mptcp_subflow_context *subflow; 20863b1d6210SPaolo Abeni struct sock *backup = NULL; 20873b1d6210SPaolo Abeni 20883b1d6210SPaolo Abeni sock_owned_by_me((const struct sock *)msk); 20893b1d6210SPaolo Abeni 2090d5f49190SPaolo Abeni if (__mptcp_check_fallback(msk)) 2091d9ca1de8SPaolo Abeni return NULL; 2092d5f49190SPaolo Abeni 20933b1d6210SPaolo Abeni mptcp_for_each_subflow(msk, subflow) { 20943b1d6210SPaolo Abeni struct sock *ssk = mptcp_subflow_tcp_sock(subflow); 20953b1d6210SPaolo Abeni 2096d5f49190SPaolo Abeni if (!mptcp_subflow_active(subflow)) 2097d5f49190SPaolo Abeni continue; 2098d5f49190SPaolo Abeni 20993b1d6210SPaolo Abeni /* still data outstanding at TCP level? Don't retransmit. */ 2100860975c6SFlorian Westphal if (!tcp_write_queue_empty(ssk)) { 2101860975c6SFlorian Westphal if (inet_csk(ssk)->icsk_ca_state >= TCP_CA_Loss) 2102860975c6SFlorian Westphal continue; 21033b1d6210SPaolo Abeni return NULL; 2104860975c6SFlorian Westphal } 21053b1d6210SPaolo Abeni 21063b1d6210SPaolo Abeni if (subflow->backup) { 21073b1d6210SPaolo Abeni if (!backup) 21083b1d6210SPaolo Abeni backup = ssk; 21093b1d6210SPaolo Abeni continue; 21103b1d6210SPaolo Abeni } 21113b1d6210SPaolo Abeni 21123b1d6210SPaolo Abeni return ssk; 21133b1d6210SPaolo Abeni } 21143b1d6210SPaolo Abeni 21153b1d6210SPaolo Abeni return backup; 21163b1d6210SPaolo Abeni } 21173b1d6210SPaolo Abeni 211817aee05dSFlorian Westphal static void mptcp_dispose_initial_subflow(struct mptcp_sock *msk) 211917aee05dSFlorian Westphal { 212017aee05dSFlorian Westphal if (msk->subflow) { 212117aee05dSFlorian Westphal iput(SOCK_INODE(msk->subflow)); 212217aee05dSFlorian Westphal msk->subflow = NULL; 212317aee05dSFlorian Westphal } 212417aee05dSFlorian Westphal } 212517aee05dSFlorian Westphal 2126cec37a6eSPeter Krystad /* subflow sockets can be either outgoing (connect) or incoming 2127cec37a6eSPeter Krystad * (accept). 2128cec37a6eSPeter Krystad * 2129cec37a6eSPeter Krystad * Outgoing subflows use in-kernel sockets. 2130cec37a6eSPeter Krystad * Incoming subflows do not have their own 'struct socket' allocated, 2131cec37a6eSPeter Krystad * so we need to use tcp_close() after detaching them from the mptcp 2132cec37a6eSPeter Krystad * parent socket. 2133cec37a6eSPeter Krystad */ 2134a141e02eSFlorian Westphal static void __mptcp_close_ssk(struct sock *sk, struct sock *ssk, 2135e16163b6SPaolo Abeni struct mptcp_subflow_context *subflow) 2136cec37a6eSPeter Krystad { 2137e0be4931SFlorian Westphal struct mptcp_sock *msk = mptcp_sk(sk); 2138e0be4931SFlorian Westphal 2139cec37a6eSPeter Krystad list_del(&subflow->node); 2140cec37a6eSPeter Krystad 21413f8b2667SPaolo Abeni lock_sock_nested(ssk, SINGLE_DEPTH_NESTING); 2142e16163b6SPaolo Abeni 2143e16163b6SPaolo Abeni /* if we are invoked by the msk cleanup code, the subflow is 2144e16163b6SPaolo Abeni * already orphaned 2145e16163b6SPaolo Abeni */ 2146866f26f2SPaolo Abeni if (ssk->sk_socket) 2147e16163b6SPaolo Abeni sock_orphan(ssk); 2148e16163b6SPaolo Abeni 2149d7b1bfd0SPaolo Abeni subflow->disposable = 1; 2150d7b1bfd0SPaolo Abeni 2151e16163b6SPaolo Abeni /* if ssk hit tcp_done(), tcp_cleanup_ulp() cleared the related ops 2152e16163b6SPaolo Abeni * the ssk has been already destroyed, we just need to release the 2153e16163b6SPaolo Abeni * reference owned by msk; 2154e16163b6SPaolo Abeni */ 2155e16163b6SPaolo Abeni if (!inet_csk(ssk)->icsk_ulp_ops) { 2156e16163b6SPaolo Abeni kfree_rcu(subflow, rcu); 2157e16163b6SPaolo Abeni } else { 2158d7b1bfd0SPaolo Abeni /* otherwise tcp will dispose of the ssk and subflow ctx */ 2159e16163b6SPaolo Abeni __tcp_close(ssk, 0); 2160e16163b6SPaolo Abeni 2161e16163b6SPaolo Abeni /* close acquired an extra ref */ 2162e16163b6SPaolo Abeni __sock_put(ssk); 2163e16163b6SPaolo Abeni } 2164e16163b6SPaolo Abeni release_sock(ssk); 2165e16163b6SPaolo Abeni 2166e16163b6SPaolo Abeni sock_put(ssk); 2167e0be4931SFlorian Westphal 2168e0be4931SFlorian Westphal if (ssk == msk->last_snd) 2169e0be4931SFlorian Westphal msk->last_snd = NULL; 217017aee05dSFlorian Westphal 2171c8fe62f0SFlorian Westphal if (ssk == msk->ack_hint) 2172c8fe62f0SFlorian Westphal msk->ack_hint = NULL; 2173c8fe62f0SFlorian Westphal 2174c8fe62f0SFlorian Westphal if (ssk == msk->first) 2175c8fe62f0SFlorian Westphal msk->first = NULL; 2176c8fe62f0SFlorian Westphal 217717aee05dSFlorian Westphal if (msk->subflow && ssk == msk->subflow->sk) 217817aee05dSFlorian Westphal mptcp_dispose_initial_subflow(msk); 2179f870fa0bSMat Martineau } 2180f870fa0bSMat Martineau 2181a141e02eSFlorian Westphal void mptcp_close_ssk(struct sock *sk, struct sock *ssk, 2182a141e02eSFlorian Westphal struct mptcp_subflow_context *subflow) 2183a141e02eSFlorian Westphal { 2184b911c97cSFlorian Westphal if (sk->sk_state == TCP_ESTABLISHED) 2185b911c97cSFlorian Westphal mptcp_event(MPTCP_EVENT_SUB_CLOSED, mptcp_sk(sk), ssk, GFP_KERNEL); 2186a141e02eSFlorian Westphal __mptcp_close_ssk(sk, ssk, subflow); 2187a141e02eSFlorian Westphal } 2188a141e02eSFlorian Westphal 2189dc24f8b4SPaolo Abeni static unsigned int mptcp_sync_mss(struct sock *sk, u32 pmtu) 2190dc24f8b4SPaolo Abeni { 2191dc24f8b4SPaolo Abeni return 0; 2192dc24f8b4SPaolo Abeni } 2193dc24f8b4SPaolo Abeni 21940e4f35d7SPaolo Abeni static void __mptcp_close_subflow(struct mptcp_sock *msk) 21950e4f35d7SPaolo Abeni { 21960e4f35d7SPaolo Abeni struct mptcp_subflow_context *subflow, *tmp; 21970e4f35d7SPaolo Abeni 21983abc05d9SFlorian Westphal might_sleep(); 21993abc05d9SFlorian Westphal 22000e4f35d7SPaolo Abeni list_for_each_entry_safe(subflow, tmp, &msk->conn_list, node) { 22010e4f35d7SPaolo Abeni struct sock *ssk = mptcp_subflow_tcp_sock(subflow); 22020e4f35d7SPaolo Abeni 22030e4f35d7SPaolo Abeni if (inet_sk_state_load(ssk) != TCP_CLOSE) 22040e4f35d7SPaolo Abeni continue; 22050e4f35d7SPaolo Abeni 220640947e13SFlorian Westphal /* 'subflow_data_ready' will re-sched once rx queue is empty */ 220740947e13SFlorian Westphal if (!skb_queue_empty_lockless(&ssk->sk_receive_queue)) 220840947e13SFlorian Westphal continue; 220940947e13SFlorian Westphal 2210a141e02eSFlorian Westphal mptcp_close_ssk((struct sock *)msk, ssk, subflow); 22110e4f35d7SPaolo Abeni } 22120e4f35d7SPaolo Abeni } 22130e4f35d7SPaolo Abeni 2214e16163b6SPaolo Abeni static bool mptcp_check_close_timeout(const struct sock *sk) 2215e16163b6SPaolo Abeni { 2216e16163b6SPaolo Abeni s32 delta = tcp_jiffies32 - inet_csk(sk)->icsk_mtup.probe_timestamp; 2217e16163b6SPaolo Abeni struct mptcp_subflow_context *subflow; 2218e16163b6SPaolo Abeni 2219e16163b6SPaolo Abeni if (delta >= TCP_TIMEWAIT_LEN) 2220e16163b6SPaolo Abeni return true; 2221e16163b6SPaolo Abeni 2222e16163b6SPaolo Abeni /* if all subflows are in closed status don't bother with additional 2223e16163b6SPaolo Abeni * timeout 2224e16163b6SPaolo Abeni */ 2225e16163b6SPaolo Abeni mptcp_for_each_subflow(mptcp_sk(sk), subflow) { 2226e16163b6SPaolo Abeni if (inet_sk_state_load(mptcp_subflow_tcp_sock(subflow)) != 2227e16163b6SPaolo Abeni TCP_CLOSE) 2228e16163b6SPaolo Abeni return false; 2229e16163b6SPaolo Abeni } 2230e16163b6SPaolo Abeni return true; 2231e16163b6SPaolo Abeni } 2232e16163b6SPaolo Abeni 223350c504a2SFlorian Westphal static void mptcp_check_fastclose(struct mptcp_sock *msk) 223450c504a2SFlorian Westphal { 223550c504a2SFlorian Westphal struct mptcp_subflow_context *subflow, *tmp; 223650c504a2SFlorian Westphal struct sock *sk = &msk->sk.icsk_inet.sk; 223750c504a2SFlorian Westphal 223850c504a2SFlorian Westphal if (likely(!READ_ONCE(msk->rcv_fastclose))) 223950c504a2SFlorian Westphal return; 224050c504a2SFlorian Westphal 224150c504a2SFlorian Westphal mptcp_token_destroy(msk); 224250c504a2SFlorian Westphal 224350c504a2SFlorian Westphal list_for_each_entry_safe(subflow, tmp, &msk->conn_list, node) { 224450c504a2SFlorian Westphal struct sock *tcp_sk = mptcp_subflow_tcp_sock(subflow); 224550c504a2SFlorian Westphal 224650c504a2SFlorian Westphal lock_sock(tcp_sk); 224750c504a2SFlorian Westphal if (tcp_sk->sk_state != TCP_CLOSE) { 224850c504a2SFlorian Westphal tcp_send_active_reset(tcp_sk, GFP_ATOMIC); 224950c504a2SFlorian Westphal tcp_set_state(tcp_sk, TCP_CLOSE); 225050c504a2SFlorian Westphal } 225150c504a2SFlorian Westphal release_sock(tcp_sk); 225250c504a2SFlorian Westphal } 225350c504a2SFlorian Westphal 225450c504a2SFlorian Westphal inet_sk_state_store(sk, TCP_CLOSE); 225550c504a2SFlorian Westphal sk->sk_shutdown = SHUTDOWN_MASK; 225650c504a2SFlorian Westphal smp_mb__before_atomic(); /* SHUTDOWN must be visible first */ 225750c504a2SFlorian Westphal set_bit(MPTCP_DATA_READY, &msk->flags); 225850c504a2SFlorian Westphal set_bit(MPTCP_WORK_CLOSE_SUBFLOW, &msk->flags); 225950c504a2SFlorian Westphal 226050c504a2SFlorian Westphal mptcp_close_wake_up(sk); 226150c504a2SFlorian Westphal } 226250c504a2SFlorian Westphal 22632948d0a1SPaolo Abeni static void __mptcp_retrans(struct sock *sk) 226480992017SPaolo Abeni { 22652948d0a1SPaolo Abeni struct mptcp_sock *msk = mptcp_sk(sk); 2266caf971dfSPaolo Abeni struct mptcp_sendmsg_info info = {}; 22673b1d6210SPaolo Abeni struct mptcp_data_frag *dfrag; 22683b1d6210SPaolo Abeni size_t copied = 0; 22692948d0a1SPaolo Abeni struct sock *ssk; 22702948d0a1SPaolo Abeni int ret; 22712948d0a1SPaolo Abeni 2272417789dfSPaolo Abeni __mptcp_clean_una_wakeup(sk); 22732948d0a1SPaolo Abeni dfrag = mptcp_rtx_head(sk); 22742948d0a1SPaolo Abeni if (!dfrag) 22752948d0a1SPaolo Abeni return; 22762948d0a1SPaolo Abeni 22772948d0a1SPaolo Abeni ssk = mptcp_subflow_get_retrans(msk); 22782948d0a1SPaolo Abeni if (!ssk) 22792948d0a1SPaolo Abeni goto reset_timer; 22802948d0a1SPaolo Abeni 22812948d0a1SPaolo Abeni lock_sock(ssk); 22822948d0a1SPaolo Abeni 22832948d0a1SPaolo Abeni /* limit retransmission to the bytes already sent on some subflows */ 22842948d0a1SPaolo Abeni info.sent = 0; 22852948d0a1SPaolo Abeni info.limit = dfrag->already_sent; 22862948d0a1SPaolo Abeni while (info.sent < dfrag->already_sent) { 22872948d0a1SPaolo Abeni if (!mptcp_alloc_tx_skb(sk, ssk)) 22882948d0a1SPaolo Abeni break; 22892948d0a1SPaolo Abeni 22902948d0a1SPaolo Abeni ret = mptcp_sendmsg_frag(sk, ssk, dfrag, &info); 22912948d0a1SPaolo Abeni if (ret <= 0) 22922948d0a1SPaolo Abeni break; 22932948d0a1SPaolo Abeni 22942948d0a1SPaolo Abeni MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_RETRANSSEGS); 22952948d0a1SPaolo Abeni copied += ret; 22962948d0a1SPaolo Abeni info.sent += ret; 22972948d0a1SPaolo Abeni } 22982948d0a1SPaolo Abeni if (copied) 22992948d0a1SPaolo Abeni tcp_push(ssk, 0, info.mss_now, tcp_sk(ssk)->nonagle, 23002948d0a1SPaolo Abeni info.size_goal); 23012948d0a1SPaolo Abeni 23022948d0a1SPaolo Abeni mptcp_set_timeout(sk, ssk); 23032948d0a1SPaolo Abeni release_sock(ssk); 23042948d0a1SPaolo Abeni 23052948d0a1SPaolo Abeni reset_timer: 23062948d0a1SPaolo Abeni if (!mptcp_timer_pending(sk)) 23072948d0a1SPaolo Abeni mptcp_reset_timer(sk); 23082948d0a1SPaolo Abeni } 23092948d0a1SPaolo Abeni 23102948d0a1SPaolo Abeni static void mptcp_worker(struct work_struct *work) 23112948d0a1SPaolo Abeni { 23122948d0a1SPaolo Abeni struct mptcp_sock *msk = container_of(work, struct mptcp_sock, work); 23132948d0a1SPaolo Abeni struct sock *sk = &msk->sk.icsk_inet.sk; 23142948d0a1SPaolo Abeni int state; 231580992017SPaolo Abeni 231680992017SPaolo Abeni lock_sock(sk); 2317e16163b6SPaolo Abeni state = sk->sk_state; 2318e16163b6SPaolo Abeni if (unlikely(state == TCP_CLOSE)) 2319e16163b6SPaolo Abeni goto unlock; 2320e16163b6SPaolo Abeni 232143b54c6eSMat Martineau mptcp_check_data_fin_ack(sk); 2322ec3edaa7SPeter Krystad __mptcp_flush_join_list(msk); 232350c504a2SFlorian Westphal 232450c504a2SFlorian Westphal mptcp_check_fastclose(msk); 232550c504a2SFlorian Westphal 2326b416268bSFlorian Westphal if (msk->pm.status) 2327e9801430SFlorian Westphal mptcp_pm_nl_work(msk); 2328b416268bSFlorian Westphal 232959832e24SFlorian Westphal if (test_and_clear_bit(MPTCP_WORK_EOF, &msk->flags)) 233059832e24SFlorian Westphal mptcp_check_for_eof(msk); 233159832e24SFlorian Westphal 23326e628cd3SPaolo Abeni __mptcp_check_send_data_fin(sk); 233343b54c6eSMat Martineau mptcp_check_data_fin(sk); 233443b54c6eSMat Martineau 2335341c6524SPaolo Abeni /* There is no point in keeping around an orphaned sk timedout or 2336341c6524SPaolo Abeni * closed, but we need the msk around to reply to incoming DATA_FIN, 2337341c6524SPaolo Abeni * even if it is orphaned and in FIN_WAIT2 state 2338e16163b6SPaolo Abeni */ 2339e16163b6SPaolo Abeni if (sock_flag(sk, SOCK_DEAD) && 2340341c6524SPaolo Abeni (mptcp_check_close_timeout(sk) || sk->sk_state == TCP_CLOSE)) { 2341e16163b6SPaolo Abeni inet_sk_state_store(sk, TCP_CLOSE); 2342e16163b6SPaolo Abeni __mptcp_destroy_sock(sk); 2343e16163b6SPaolo Abeni goto unlock; 2344e16163b6SPaolo Abeni } 2345e16163b6SPaolo Abeni 2346b263b0d7SFlorian Westphal if (test_and_clear_bit(MPTCP_WORK_CLOSE_SUBFLOW, &msk->flags)) 2347b263b0d7SFlorian Westphal __mptcp_close_subflow(msk); 2348b263b0d7SFlorian Westphal 23492948d0a1SPaolo Abeni if (test_and_clear_bit(MPTCP_WORK_RTX, &msk->flags)) 23502948d0a1SPaolo Abeni __mptcp_retrans(sk); 23513b1d6210SPaolo Abeni 23523b1d6210SPaolo Abeni unlock: 235380992017SPaolo Abeni release_sock(sk); 235480992017SPaolo Abeni sock_put(sk); 235580992017SPaolo Abeni } 235680992017SPaolo Abeni 2357784325e9SMatthieu Baerts static int __mptcp_init_sock(struct sock *sk) 2358f870fa0bSMat Martineau { 2359cec37a6eSPeter Krystad struct mptcp_sock *msk = mptcp_sk(sk); 2360cec37a6eSPeter Krystad 2361ec3edaa7SPeter Krystad spin_lock_init(&msk->join_list_lock); 2362ec3edaa7SPeter Krystad 2363cec37a6eSPeter Krystad INIT_LIST_HEAD(&msk->conn_list); 2364ec3edaa7SPeter Krystad INIT_LIST_HEAD(&msk->join_list); 236518b683bfSPaolo Abeni INIT_LIST_HEAD(&msk->rtx_queue); 236680992017SPaolo Abeni INIT_WORK(&msk->work, mptcp_worker); 236787952603SPaolo Abeni __skb_queue_head_init(&msk->receive_queue); 2368724cfd2eSPaolo Abeni __skb_queue_head_init(&msk->skb_tx_cache); 2369ab174ad8SPaolo Abeni msk->out_of_order_queue = RB_ROOT; 2370f0e6a4cfSPaolo Abeni msk->first_pending = NULL; 2371e93da928SPaolo Abeni msk->wmem_reserved = 0; 237287952603SPaolo Abeni msk->rmem_released = 0; 2373724cfd2eSPaolo Abeni msk->tx_pending_data = 0; 2374724cfd2eSPaolo Abeni msk->size_goal_cache = TCP_BASE_MSS; 2375cec37a6eSPeter Krystad 2376ea4ca586SPaolo Abeni msk->ack_hint = NULL; 23778ab183deSPaolo Abeni msk->first = NULL; 2378dc24f8b4SPaolo Abeni inet_csk(sk)->icsk_sync_mss = mptcp_sync_mss; 23798ab183deSPaolo Abeni 23801b1c7a0eSPeter Krystad mptcp_pm_data_init(msk); 23811b1c7a0eSPeter Krystad 2382b51f9b80SPaolo Abeni /* re-use the csk retrans timer for MPTCP-level retrans */ 2383b51f9b80SPaolo Abeni timer_setup(&msk->sk.icsk_retransmit_timer, mptcp_retransmit_timer, 0); 2384e16163b6SPaolo Abeni timer_setup(&sk->sk_timer, mptcp_timeout_timer, 0); 2385f870fa0bSMat Martineau return 0; 2386f870fa0bSMat Martineau } 2387f870fa0bSMat Martineau 2388784325e9SMatthieu Baerts static int mptcp_init_sock(struct sock *sk) 2389784325e9SMatthieu Baerts { 2390fc518953SFlorian Westphal struct net *net = sock_net(sk); 2391fc518953SFlorian Westphal int ret; 239218b683bfSPaolo Abeni 2393b6c08380SGeliang Tang ret = __mptcp_init_sock(sk); 2394b6c08380SGeliang Tang if (ret) 2395b6c08380SGeliang Tang return ret; 2396b6c08380SGeliang Tang 2397fc518953SFlorian Westphal if (!mptcp_is_enabled(net)) 2398fc518953SFlorian Westphal return -ENOPROTOOPT; 2399fc518953SFlorian Westphal 2400fc518953SFlorian Westphal if (unlikely(!net->mib.mptcp_statistics) && !mptcp_mib_alloc(net)) 2401fc518953SFlorian Westphal return -ENOMEM; 2402fc518953SFlorian Westphal 2403fa68018dSPaolo Abeni ret = __mptcp_socket_create(mptcp_sk(sk)); 2404fa68018dSPaolo Abeni if (ret) 2405fa68018dSPaolo Abeni return ret; 2406fa68018dSPaolo Abeni 2407d027236cSPaolo Abeni sk_sockets_allocated_inc(sk); 2408a6b118feSFlorian Westphal sk->sk_rcvbuf = sock_net(sk)->ipv4.sysctl_tcp_rmem[1]; 2409da51aef5SPaolo Abeni sk->sk_sndbuf = sock_net(sk)->ipv4.sysctl_tcp_wmem[1]; 2410d027236cSPaolo Abeni 241118b683bfSPaolo Abeni return 0; 241218b683bfSPaolo Abeni } 241318b683bfSPaolo Abeni 241418b683bfSPaolo Abeni static void __mptcp_clear_xmit(struct sock *sk) 241518b683bfSPaolo Abeni { 241618b683bfSPaolo Abeni struct mptcp_sock *msk = mptcp_sk(sk); 241718b683bfSPaolo Abeni struct mptcp_data_frag *dtmp, *dfrag; 2418724cfd2eSPaolo Abeni struct sk_buff *skb; 241918b683bfSPaolo Abeni 2420d9ca1de8SPaolo Abeni WRITE_ONCE(msk->first_pending, NULL); 242118b683bfSPaolo Abeni list_for_each_entry_safe(dfrag, dtmp, &msk->rtx_queue, list) 2422d027236cSPaolo Abeni dfrag_clear(sk, dfrag); 2423724cfd2eSPaolo Abeni while ((skb = __skb_dequeue(&msk->skb_tx_cache)) != NULL) { 2424724cfd2eSPaolo Abeni sk->sk_forward_alloc += skb->truesize; 2425724cfd2eSPaolo Abeni kfree_skb(skb); 2426724cfd2eSPaolo Abeni } 2427784325e9SMatthieu Baerts } 2428784325e9SMatthieu Baerts 242980992017SPaolo Abeni static void mptcp_cancel_work(struct sock *sk) 243080992017SPaolo Abeni { 243180992017SPaolo Abeni struct mptcp_sock *msk = mptcp_sk(sk); 243280992017SPaolo Abeni 2433b2771d24SPaolo Abeni if (cancel_work_sync(&msk->work)) 2434e16163b6SPaolo Abeni __sock_put(sk); 243580992017SPaolo Abeni } 243680992017SPaolo Abeni 2437d0876b22SGeliang Tang void mptcp_subflow_shutdown(struct sock *sk, struct sock *ssk, int how) 243821498490SPeter Krystad { 243921498490SPeter Krystad lock_sock(ssk); 244021498490SPeter Krystad 244121498490SPeter Krystad switch (ssk->sk_state) { 244221498490SPeter Krystad case TCP_LISTEN: 244321498490SPeter Krystad if (!(how & RCV_SHUTDOWN)) 244421498490SPeter Krystad break; 2445df561f66SGustavo A. R. Silva fallthrough; 244621498490SPeter Krystad case TCP_SYN_SENT: 244721498490SPeter Krystad tcp_disconnect(ssk, O_NONBLOCK); 244821498490SPeter Krystad break; 244921498490SPeter Krystad default: 245043b54c6eSMat Martineau if (__mptcp_check_fallback(mptcp_sk(sk))) { 245143b54c6eSMat Martineau pr_debug("Fallback"); 245221498490SPeter Krystad ssk->sk_shutdown |= how; 245321498490SPeter Krystad tcp_shutdown(ssk, how); 245443b54c6eSMat Martineau } else { 245543b54c6eSMat Martineau pr_debug("Sending DATA_FIN on subflow %p", ssk); 245643b54c6eSMat Martineau mptcp_set_timeout(sk, ssk); 245743b54c6eSMat Martineau tcp_send_ack(ssk); 245843b54c6eSMat Martineau } 245921498490SPeter Krystad break; 246021498490SPeter Krystad } 246121498490SPeter Krystad 246221498490SPeter Krystad release_sock(ssk); 246321498490SPeter Krystad } 246421498490SPeter Krystad 24656920b851SMat Martineau static const unsigned char new_state[16] = { 24666920b851SMat Martineau /* current state: new state: action: */ 24676920b851SMat Martineau [0 /* (Invalid) */] = TCP_CLOSE, 24686920b851SMat Martineau [TCP_ESTABLISHED] = TCP_FIN_WAIT1 | TCP_ACTION_FIN, 24696920b851SMat Martineau [TCP_SYN_SENT] = TCP_CLOSE, 24706920b851SMat Martineau [TCP_SYN_RECV] = TCP_FIN_WAIT1 | TCP_ACTION_FIN, 24716920b851SMat Martineau [TCP_FIN_WAIT1] = TCP_FIN_WAIT1, 24726920b851SMat Martineau [TCP_FIN_WAIT2] = TCP_FIN_WAIT2, 24736920b851SMat Martineau [TCP_TIME_WAIT] = TCP_CLOSE, /* should not happen ! */ 24746920b851SMat Martineau [TCP_CLOSE] = TCP_CLOSE, 24756920b851SMat Martineau [TCP_CLOSE_WAIT] = TCP_LAST_ACK | TCP_ACTION_FIN, 24766920b851SMat Martineau [TCP_LAST_ACK] = TCP_LAST_ACK, 24776920b851SMat Martineau [TCP_LISTEN] = TCP_CLOSE, 24786920b851SMat Martineau [TCP_CLOSING] = TCP_CLOSING, 24796920b851SMat Martineau [TCP_NEW_SYN_RECV] = TCP_CLOSE, /* should not happen ! */ 24806920b851SMat Martineau }; 24816920b851SMat Martineau 24826920b851SMat Martineau static int mptcp_close_state(struct sock *sk) 24836920b851SMat Martineau { 24846920b851SMat Martineau int next = (int)new_state[sk->sk_state]; 24856920b851SMat Martineau int ns = next & TCP_STATE_MASK; 24866920b851SMat Martineau 24876920b851SMat Martineau inet_sk_state_store(sk, ns); 24886920b851SMat Martineau 24896920b851SMat Martineau return next & TCP_ACTION_FIN; 24906920b851SMat Martineau } 24916920b851SMat Martineau 2492e16163b6SPaolo Abeni static void __mptcp_check_send_data_fin(struct sock *sk) 2493e16163b6SPaolo Abeni { 2494e16163b6SPaolo Abeni struct mptcp_subflow_context *subflow; 2495e16163b6SPaolo Abeni struct mptcp_sock *msk = mptcp_sk(sk); 2496e16163b6SPaolo Abeni 2497e16163b6SPaolo Abeni pr_debug("msk=%p snd_data_fin_enable=%d pending=%d snd_nxt=%llu write_seq=%llu", 2498e16163b6SPaolo Abeni msk, msk->snd_data_fin_enable, !!mptcp_send_head(sk), 2499e16163b6SPaolo Abeni msk->snd_nxt, msk->write_seq); 2500e16163b6SPaolo Abeni 2501e16163b6SPaolo Abeni /* we still need to enqueue subflows or not really shutting down, 2502e16163b6SPaolo Abeni * skip this 2503e16163b6SPaolo Abeni */ 2504e16163b6SPaolo Abeni if (!msk->snd_data_fin_enable || msk->snd_nxt + 1 != msk->write_seq || 2505e16163b6SPaolo Abeni mptcp_send_head(sk)) 2506e16163b6SPaolo Abeni return; 2507e16163b6SPaolo Abeni 2508e16163b6SPaolo Abeni WRITE_ONCE(msk->snd_nxt, msk->write_seq); 2509e16163b6SPaolo Abeni 251026aa2314SPaolo Abeni /* fallback socket will not get data_fin/ack, can move to the next 251126aa2314SPaolo Abeni * state now 251226aa2314SPaolo Abeni */ 251326aa2314SPaolo Abeni if (__mptcp_check_fallback(msk)) { 251426aa2314SPaolo Abeni if ((1 << sk->sk_state) & (TCPF_CLOSING | TCPF_LAST_ACK)) { 2515e16163b6SPaolo Abeni inet_sk_state_store(sk, TCP_CLOSE); 2516e16163b6SPaolo Abeni mptcp_close_wake_up(sk); 251726aa2314SPaolo Abeni } else if (sk->sk_state == TCP_FIN_WAIT1) { 251826aa2314SPaolo Abeni inet_sk_state_store(sk, TCP_FIN_WAIT2); 251926aa2314SPaolo Abeni } 2520e16163b6SPaolo Abeni } 2521e16163b6SPaolo Abeni 2522e16163b6SPaolo Abeni __mptcp_flush_join_list(msk); 2523e16163b6SPaolo Abeni mptcp_for_each_subflow(msk, subflow) { 2524e16163b6SPaolo Abeni struct sock *tcp_sk = mptcp_subflow_tcp_sock(subflow); 2525e16163b6SPaolo Abeni 2526e16163b6SPaolo Abeni mptcp_subflow_shutdown(sk, tcp_sk, SEND_SHUTDOWN); 2527e16163b6SPaolo Abeni } 2528e16163b6SPaolo Abeni } 2529e16163b6SPaolo Abeni 2530e16163b6SPaolo Abeni static void __mptcp_wr_shutdown(struct sock *sk) 2531e16163b6SPaolo Abeni { 2532e16163b6SPaolo Abeni struct mptcp_sock *msk = mptcp_sk(sk); 2533e16163b6SPaolo Abeni 2534e16163b6SPaolo Abeni pr_debug("msk=%p snd_data_fin_enable=%d shutdown=%x state=%d pending=%d", 2535e16163b6SPaolo Abeni msk, msk->snd_data_fin_enable, sk->sk_shutdown, sk->sk_state, 2536e16163b6SPaolo Abeni !!mptcp_send_head(sk)); 2537e16163b6SPaolo Abeni 2538e16163b6SPaolo Abeni /* will be ignored by fallback sockets */ 2539e16163b6SPaolo Abeni WRITE_ONCE(msk->write_seq, msk->write_seq + 1); 2540e16163b6SPaolo Abeni WRITE_ONCE(msk->snd_data_fin_enable, 1); 2541e16163b6SPaolo Abeni 2542e16163b6SPaolo Abeni __mptcp_check_send_data_fin(sk); 2543e16163b6SPaolo Abeni } 2544e16163b6SPaolo Abeni 2545e16163b6SPaolo Abeni static void __mptcp_destroy_sock(struct sock *sk) 2546f870fa0bSMat Martineau { 2547cec37a6eSPeter Krystad struct mptcp_subflow_context *subflow, *tmp; 2548f870fa0bSMat Martineau struct mptcp_sock *msk = mptcp_sk(sk); 2549b2c5b614SFlorian Westphal LIST_HEAD(conn_list); 2550f870fa0bSMat Martineau 2551e16163b6SPaolo Abeni pr_debug("msk=%p", msk); 25522c22c06cSFlorian Westphal 25533abc05d9SFlorian Westphal might_sleep(); 25543abc05d9SFlorian Westphal 255510f6d46cSPaolo Abeni /* be sure to always acquire the join list lock, to sync vs 255610f6d46cSPaolo Abeni * mptcp_finish_join(). 255710f6d46cSPaolo Abeni */ 255810f6d46cSPaolo Abeni spin_lock_bh(&msk->join_list_lock); 255910f6d46cSPaolo Abeni list_splice_tail_init(&msk->join_list, &msk->conn_list); 256010f6d46cSPaolo Abeni spin_unlock_bh(&msk->join_list_lock); 2561b2c5b614SFlorian Westphal list_splice_init(&msk->conn_list, &conn_list); 2562b2c5b614SFlorian Westphal 25636e628cd3SPaolo Abeni sk_stop_timer(sk, &msk->sk.icsk_retransmit_timer); 2564e16163b6SPaolo Abeni sk_stop_timer(sk, &sk->sk_timer); 2565e16163b6SPaolo Abeni msk->pm.status = 0; 2566b2c5b614SFlorian Westphal 2567b2c5b614SFlorian Westphal list_for_each_entry_safe(subflow, tmp, &conn_list, node) { 2568cec37a6eSPeter Krystad struct sock *ssk = mptcp_subflow_tcp_sock(subflow); 2569e16163b6SPaolo Abeni __mptcp_close_ssk(sk, ssk, subflow); 2570f870fa0bSMat Martineau } 2571f870fa0bSMat Martineau 2572e16163b6SPaolo Abeni sk->sk_prot->destroy(sk); 2573e16163b6SPaolo Abeni 2574e93da928SPaolo Abeni WARN_ON_ONCE(msk->wmem_reserved); 257587952603SPaolo Abeni WARN_ON_ONCE(msk->rmem_released); 2576e16163b6SPaolo Abeni sk_stream_kill_queues(sk); 2577e16163b6SPaolo Abeni xfrm_sk_free_policy(sk); 2578e16163b6SPaolo Abeni sk_refcnt_debug_release(sk); 257917aee05dSFlorian Westphal mptcp_dispose_initial_subflow(msk); 2580e16163b6SPaolo Abeni sock_put(sk); 2581e16163b6SPaolo Abeni } 2582e16163b6SPaolo Abeni 2583e16163b6SPaolo Abeni static void mptcp_close(struct sock *sk, long timeout) 2584e16163b6SPaolo Abeni { 2585e16163b6SPaolo Abeni struct mptcp_subflow_context *subflow; 2586e16163b6SPaolo Abeni bool do_cancel_work = false; 2587e16163b6SPaolo Abeni 2588e16163b6SPaolo Abeni lock_sock(sk); 2589e16163b6SPaolo Abeni sk->sk_shutdown = SHUTDOWN_MASK; 2590e16163b6SPaolo Abeni 2591e16163b6SPaolo Abeni if ((1 << sk->sk_state) & (TCPF_LISTEN | TCPF_CLOSE)) { 2592e16163b6SPaolo Abeni inet_sk_state_store(sk, TCP_CLOSE); 2593e16163b6SPaolo Abeni goto cleanup; 2594e16163b6SPaolo Abeni } 2595e16163b6SPaolo Abeni 2596e16163b6SPaolo Abeni if (mptcp_close_state(sk)) 2597e16163b6SPaolo Abeni __mptcp_wr_shutdown(sk); 2598e16163b6SPaolo Abeni 2599e16163b6SPaolo Abeni sk_stream_wait_close(sk, timeout); 2600e16163b6SPaolo Abeni 2601e16163b6SPaolo Abeni cleanup: 2602e16163b6SPaolo Abeni /* orphan all the subflows */ 2603e16163b6SPaolo Abeni inet_csk(sk)->icsk_mtup.probe_timestamp = tcp_jiffies32; 2604e16163b6SPaolo Abeni list_for_each_entry(subflow, &mptcp_sk(sk)->conn_list, node) { 2605e16163b6SPaolo Abeni struct sock *ssk = mptcp_subflow_tcp_sock(subflow); 2606866f26f2SPaolo Abeni bool slow = lock_sock_fast(ssk); 2607e16163b6SPaolo Abeni 2608e16163b6SPaolo Abeni sock_orphan(ssk); 2609e16163b6SPaolo Abeni unlock_sock_fast(ssk, slow); 2610e16163b6SPaolo Abeni } 2611e16163b6SPaolo Abeni sock_orphan(sk); 2612e16163b6SPaolo Abeni 2613e16163b6SPaolo Abeni sock_hold(sk); 2614e16163b6SPaolo Abeni pr_debug("msk=%p state=%d", sk, sk->sk_state); 2615e16163b6SPaolo Abeni if (sk->sk_state == TCP_CLOSE) { 2616e16163b6SPaolo Abeni __mptcp_destroy_sock(sk); 2617e16163b6SPaolo Abeni do_cancel_work = true; 2618e16163b6SPaolo Abeni } else { 2619e16163b6SPaolo Abeni sk_reset_timer(sk, &sk->sk_timer, jiffies + TCP_TIMEWAIT_LEN); 2620e16163b6SPaolo Abeni } 2621e16163b6SPaolo Abeni release_sock(sk); 2622e16163b6SPaolo Abeni if (do_cancel_work) 262380992017SPaolo Abeni mptcp_cancel_work(sk); 2624b911c97cSFlorian Westphal 2625b911c97cSFlorian Westphal if (mptcp_sk(sk)->token) 2626b911c97cSFlorian Westphal mptcp_event(MPTCP_EVENT_CLOSED, mptcp_sk(sk), NULL, GFP_KERNEL); 2627b911c97cSFlorian Westphal 2628e16163b6SPaolo Abeni sock_put(sk); 2629f870fa0bSMat Martineau } 2630f870fa0bSMat Martineau 2631cf7da0d6SPeter Krystad static void mptcp_copy_inaddrs(struct sock *msk, const struct sock *ssk) 2632cf7da0d6SPeter Krystad { 2633cf7da0d6SPeter Krystad #if IS_ENABLED(CONFIG_MPTCP_IPV6) 2634cf7da0d6SPeter Krystad const struct ipv6_pinfo *ssk6 = inet6_sk(ssk); 2635cf7da0d6SPeter Krystad struct ipv6_pinfo *msk6 = inet6_sk(msk); 2636cf7da0d6SPeter Krystad 2637cf7da0d6SPeter Krystad msk->sk_v6_daddr = ssk->sk_v6_daddr; 2638cf7da0d6SPeter Krystad msk->sk_v6_rcv_saddr = ssk->sk_v6_rcv_saddr; 2639cf7da0d6SPeter Krystad 2640cf7da0d6SPeter Krystad if (msk6 && ssk6) { 2641cf7da0d6SPeter Krystad msk6->saddr = ssk6->saddr; 2642cf7da0d6SPeter Krystad msk6->flow_label = ssk6->flow_label; 2643cf7da0d6SPeter Krystad } 2644cf7da0d6SPeter Krystad #endif 2645cf7da0d6SPeter Krystad 2646cf7da0d6SPeter Krystad inet_sk(msk)->inet_num = inet_sk(ssk)->inet_num; 2647cf7da0d6SPeter Krystad inet_sk(msk)->inet_dport = inet_sk(ssk)->inet_dport; 2648cf7da0d6SPeter Krystad inet_sk(msk)->inet_sport = inet_sk(ssk)->inet_sport; 2649cf7da0d6SPeter Krystad inet_sk(msk)->inet_daddr = inet_sk(ssk)->inet_daddr; 2650cf7da0d6SPeter Krystad inet_sk(msk)->inet_saddr = inet_sk(ssk)->inet_saddr; 2651cf7da0d6SPeter Krystad inet_sk(msk)->inet_rcv_saddr = inet_sk(ssk)->inet_rcv_saddr; 2652cf7da0d6SPeter Krystad } 2653cf7da0d6SPeter Krystad 265418b683bfSPaolo Abeni static int mptcp_disconnect(struct sock *sk, int flags) 265518b683bfSPaolo Abeni { 265676e2a55dSPaolo Abeni struct mptcp_subflow_context *subflow; 265776e2a55dSPaolo Abeni struct mptcp_sock *msk = mptcp_sk(sk); 265876e2a55dSPaolo Abeni 265976e2a55dSPaolo Abeni __mptcp_flush_join_list(msk); 266013a9499eSPaolo Abeni mptcp_for_each_subflow(msk, subflow) { 266113a9499eSPaolo Abeni struct sock *ssk = mptcp_subflow_tcp_sock(subflow); 266213a9499eSPaolo Abeni 266313a9499eSPaolo Abeni lock_sock(ssk); 266413a9499eSPaolo Abeni tcp_disconnect(ssk, flags); 266513a9499eSPaolo Abeni release_sock(ssk); 266613a9499eSPaolo Abeni } 266742c556feSFlorian Westphal return 0; 266818b683bfSPaolo Abeni } 266918b683bfSPaolo Abeni 2670b0519de8SFlorian Westphal #if IS_ENABLED(CONFIG_MPTCP_IPV6) 2671b0519de8SFlorian Westphal static struct ipv6_pinfo *mptcp_inet6_sk(const struct sock *sk) 2672b0519de8SFlorian Westphal { 2673b0519de8SFlorian Westphal unsigned int offset = sizeof(struct mptcp6_sock) - sizeof(struct ipv6_pinfo); 2674b0519de8SFlorian Westphal 2675b0519de8SFlorian Westphal return (struct ipv6_pinfo *)(((u8 *)sk) + offset); 2676b0519de8SFlorian Westphal } 2677b0519de8SFlorian Westphal #endif 2678b0519de8SFlorian Westphal 2679fca5c82cSPaolo Abeni struct sock *mptcp_sk_clone(const struct sock *sk, 2680cfde141eSPaolo Abeni const struct mptcp_options_received *mp_opt, 2681fca5c82cSPaolo Abeni struct request_sock *req) 2682b0519de8SFlorian Westphal { 268358b09919SPaolo Abeni struct mptcp_subflow_request_sock *subflow_req = mptcp_subflow_rsk(req); 2684b0519de8SFlorian Westphal struct sock *nsk = sk_clone_lock(sk, GFP_ATOMIC); 268558b09919SPaolo Abeni struct mptcp_sock *msk; 268658b09919SPaolo Abeni u64 ack_seq; 2687b0519de8SFlorian Westphal 2688b0519de8SFlorian Westphal if (!nsk) 2689b0519de8SFlorian Westphal return NULL; 2690b0519de8SFlorian Westphal 2691b0519de8SFlorian Westphal #if IS_ENABLED(CONFIG_MPTCP_IPV6) 2692b0519de8SFlorian Westphal if (nsk->sk_family == AF_INET6) 2693b0519de8SFlorian Westphal inet_sk(nsk)->pinet6 = mptcp_inet6_sk(nsk); 2694b0519de8SFlorian Westphal #endif 2695b0519de8SFlorian Westphal 269658b09919SPaolo Abeni __mptcp_init_sock(nsk); 269758b09919SPaolo Abeni 269858b09919SPaolo Abeni msk = mptcp_sk(nsk); 269958b09919SPaolo Abeni msk->local_key = subflow_req->local_key; 270058b09919SPaolo Abeni msk->token = subflow_req->token; 270158b09919SPaolo Abeni msk->subflow = NULL; 2702b93df08cSPaolo Abeni WRITE_ONCE(msk->fully_established, false); 270358b09919SPaolo Abeni 270458b09919SPaolo Abeni msk->write_seq = subflow_req->idsn + 1; 2705eaa2ffabSPaolo Abeni msk->snd_nxt = msk->write_seq; 27067439d687SPaolo Abeni msk->snd_una = msk->write_seq; 27077439d687SPaolo Abeni msk->wnd_end = msk->snd_nxt + req->rsk_rcv_wnd; 27086f8a612aSFlorian Westphal 2709cfde141eSPaolo Abeni if (mp_opt->mp_capable) { 271058b09919SPaolo Abeni msk->can_ack = true; 2711cfde141eSPaolo Abeni msk->remote_key = mp_opt->sndr_key; 271258b09919SPaolo Abeni mptcp_crypto_key_sha(msk->remote_key, NULL, &ack_seq); 271358b09919SPaolo Abeni ack_seq++; 2714917944daSMat Martineau WRITE_ONCE(msk->ack_seq, ack_seq); 2715fa3fe2b1SFlorian Westphal WRITE_ONCE(msk->rcv_wnd_sent, ack_seq); 271658b09919SPaolo Abeni } 27177f20d5fcSPaolo Abeni 27185e20087dSFlorian Westphal sock_reset_flag(nsk, SOCK_RCU_FREE); 27197f20d5fcSPaolo Abeni /* will be fully established after successful MPC subflow creation */ 27207f20d5fcSPaolo Abeni inet_sk_state_store(nsk, TCP_SYN_RECV); 27210c148460SPaolo Abeni 27220c148460SPaolo Abeni security_inet_csk_clone(nsk, req); 272358b09919SPaolo Abeni bh_unlock_sock(nsk); 272458b09919SPaolo Abeni 272558b09919SPaolo Abeni /* keep a single reference */ 272658b09919SPaolo Abeni __sock_put(nsk); 2727b0519de8SFlorian Westphal return nsk; 2728b0519de8SFlorian Westphal } 2729b0519de8SFlorian Westphal 2730a6b118feSFlorian Westphal void mptcp_rcv_space_init(struct mptcp_sock *msk, const struct sock *ssk) 2731a6b118feSFlorian Westphal { 2732a6b118feSFlorian Westphal const struct tcp_sock *tp = tcp_sk(ssk); 2733a6b118feSFlorian Westphal 2734a6b118feSFlorian Westphal msk->rcvq_space.copied = 0; 2735a6b118feSFlorian Westphal msk->rcvq_space.rtt_us = 0; 2736a6b118feSFlorian Westphal 2737a6b118feSFlorian Westphal msk->rcvq_space.time = tp->tcp_mstamp; 2738a6b118feSFlorian Westphal 2739a6b118feSFlorian Westphal /* initial rcv_space offering made to peer */ 2740a6b118feSFlorian Westphal msk->rcvq_space.space = min_t(u32, tp->rcv_wnd, 2741a6b118feSFlorian Westphal TCP_INIT_CWND * tp->advmss); 2742a6b118feSFlorian Westphal if (msk->rcvq_space.space == 0) 2743a6b118feSFlorian Westphal msk->rcvq_space.space = TCP_INIT_CWND * TCP_MSS_DEFAULT; 27446f8a612aSFlorian Westphal 27457439d687SPaolo Abeni WRITE_ONCE(msk->wnd_end, msk->snd_nxt + tcp_sk(ssk)->snd_wnd); 2746a6b118feSFlorian Westphal } 2747a6b118feSFlorian Westphal 2748cf7da0d6SPeter Krystad static struct sock *mptcp_accept(struct sock *sk, int flags, int *err, 2749cf7da0d6SPeter Krystad bool kern) 2750cf7da0d6SPeter Krystad { 2751cf7da0d6SPeter Krystad struct mptcp_sock *msk = mptcp_sk(sk); 2752cf7da0d6SPeter Krystad struct socket *listener; 2753cf7da0d6SPeter Krystad struct sock *newsk; 2754cf7da0d6SPeter Krystad 2755cf7da0d6SPeter Krystad listener = __mptcp_nmpc_socket(msk); 2756cf7da0d6SPeter Krystad if (WARN_ON_ONCE(!listener)) { 2757cf7da0d6SPeter Krystad *err = -EINVAL; 2758cf7da0d6SPeter Krystad return NULL; 2759cf7da0d6SPeter Krystad } 2760cf7da0d6SPeter Krystad 2761cf7da0d6SPeter Krystad pr_debug("msk=%p, listener=%p", msk, mptcp_subflow_ctx(listener->sk)); 2762cf7da0d6SPeter Krystad newsk = inet_csk_accept(listener->sk, flags, err, kern); 2763cf7da0d6SPeter Krystad if (!newsk) 2764cf7da0d6SPeter Krystad return NULL; 2765cf7da0d6SPeter Krystad 2766cf7da0d6SPeter Krystad pr_debug("msk=%p, subflow is mptcp=%d", msk, sk_is_mptcp(newsk)); 2767cf7da0d6SPeter Krystad if (sk_is_mptcp(newsk)) { 2768cf7da0d6SPeter Krystad struct mptcp_subflow_context *subflow; 2769cf7da0d6SPeter Krystad struct sock *new_mptcp_sock; 2770cf7da0d6SPeter Krystad 2771cf7da0d6SPeter Krystad subflow = mptcp_subflow_ctx(newsk); 277258b09919SPaolo Abeni new_mptcp_sock = subflow->conn; 277358b09919SPaolo Abeni 277458b09919SPaolo Abeni /* is_mptcp should be false if subflow->conn is missing, see 277558b09919SPaolo Abeni * subflow_syn_recv_sock() 277658b09919SPaolo Abeni */ 277758b09919SPaolo Abeni if (WARN_ON_ONCE(!new_mptcp_sock)) { 277858b09919SPaolo Abeni tcp_sk(newsk)->is_mptcp = 0; 277958b09919SPaolo Abeni return newsk; 278058b09919SPaolo Abeni } 278158b09919SPaolo Abeni 278258b09919SPaolo Abeni /* acquire the 2nd reference for the owning socket */ 278358b09919SPaolo Abeni sock_hold(new_mptcp_sock); 2784cf7da0d6SPeter Krystad newsk = new_mptcp_sock; 27850397c6d8SPaolo Abeni MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_MPCAPABLEPASSIVEACK); 2786fc518953SFlorian Westphal } else { 2787fc518953SFlorian Westphal MPTCP_INC_STATS(sock_net(sk), 2788fc518953SFlorian Westphal MPTCP_MIB_MPCAPABLEPASSIVEFALLBACK); 2789cf7da0d6SPeter Krystad } 2790cf7da0d6SPeter Krystad 2791cf7da0d6SPeter Krystad return newsk; 2792cf7da0d6SPeter Krystad } 2793cf7da0d6SPeter Krystad 27945c8c1640SGeliang Tang void mptcp_destroy_common(struct mptcp_sock *msk) 27955c8c1640SGeliang Tang { 279687952603SPaolo Abeni struct sock *sk = (struct sock *)msk; 279787952603SPaolo Abeni 27986e628cd3SPaolo Abeni __mptcp_clear_xmit(sk); 27996e628cd3SPaolo Abeni 280087952603SPaolo Abeni /* move to sk_receive_queue, sk_stream_kill_queues will purge it */ 280187952603SPaolo Abeni skb_queue_splice_tail_init(&msk->receive_queue, &sk->sk_receive_queue); 280287952603SPaolo Abeni 28035c8c1640SGeliang Tang skb_rbtree_purge(&msk->out_of_order_queue); 28045c8c1640SGeliang Tang mptcp_token_destroy(msk); 28055c8c1640SGeliang Tang mptcp_pm_free_anno_list(msk); 28065c8c1640SGeliang Tang } 28075c8c1640SGeliang Tang 280879c0949eSPeter Krystad static void mptcp_destroy(struct sock *sk) 280979c0949eSPeter Krystad { 2810c9fd9c5fSFlorian Westphal struct mptcp_sock *msk = mptcp_sk(sk); 2811c9fd9c5fSFlorian Westphal 28125c8c1640SGeliang Tang mptcp_destroy_common(msk); 2813d027236cSPaolo Abeni sk_sockets_allocated_dec(sk); 281479c0949eSPeter Krystad } 281579c0949eSPeter Krystad 2816fd1452d8SFlorian Westphal static int mptcp_setsockopt_sol_socket(struct mptcp_sock *msk, int optname, 2817a7b75c5aSChristoph Hellwig sockptr_t optval, unsigned int optlen) 2818fd1452d8SFlorian Westphal { 2819fd1452d8SFlorian Westphal struct sock *sk = (struct sock *)msk; 2820fd1452d8SFlorian Westphal struct socket *ssock; 2821fd1452d8SFlorian Westphal int ret; 2822fd1452d8SFlorian Westphal 2823fd1452d8SFlorian Westphal switch (optname) { 2824fd1452d8SFlorian Westphal case SO_REUSEPORT: 2825fd1452d8SFlorian Westphal case SO_REUSEADDR: 2826fd1452d8SFlorian Westphal lock_sock(sk); 2827fd1452d8SFlorian Westphal ssock = __mptcp_nmpc_socket(msk); 2828fd1452d8SFlorian Westphal if (!ssock) { 2829fd1452d8SFlorian Westphal release_sock(sk); 2830fd1452d8SFlorian Westphal return -EINVAL; 2831fd1452d8SFlorian Westphal } 2832fd1452d8SFlorian Westphal 2833a7b75c5aSChristoph Hellwig ret = sock_setsockopt(ssock, SOL_SOCKET, optname, optval, optlen); 2834fd1452d8SFlorian Westphal if (ret == 0) { 2835fd1452d8SFlorian Westphal if (optname == SO_REUSEPORT) 2836fd1452d8SFlorian Westphal sk->sk_reuseport = ssock->sk->sk_reuseport; 2837fd1452d8SFlorian Westphal else if (optname == SO_REUSEADDR) 2838fd1452d8SFlorian Westphal sk->sk_reuse = ssock->sk->sk_reuse; 2839fd1452d8SFlorian Westphal } 2840fd1452d8SFlorian Westphal release_sock(sk); 2841fd1452d8SFlorian Westphal return ret; 2842fd1452d8SFlorian Westphal } 2843fd1452d8SFlorian Westphal 2844a7b75c5aSChristoph Hellwig return sock_setsockopt(sk->sk_socket, SOL_SOCKET, optname, optval, optlen); 2845fd1452d8SFlorian Westphal } 2846fd1452d8SFlorian Westphal 2847c9b95a13SFlorian Westphal static int mptcp_setsockopt_v6(struct mptcp_sock *msk, int optname, 2848a7b75c5aSChristoph Hellwig sockptr_t optval, unsigned int optlen) 2849c9b95a13SFlorian Westphal { 2850c9b95a13SFlorian Westphal struct sock *sk = (struct sock *)msk; 2851c9b95a13SFlorian Westphal int ret = -EOPNOTSUPP; 2852c9b95a13SFlorian Westphal struct socket *ssock; 2853c9b95a13SFlorian Westphal 2854c9b95a13SFlorian Westphal switch (optname) { 2855c9b95a13SFlorian Westphal case IPV6_V6ONLY: 2856c9b95a13SFlorian Westphal lock_sock(sk); 2857c9b95a13SFlorian Westphal ssock = __mptcp_nmpc_socket(msk); 2858c9b95a13SFlorian Westphal if (!ssock) { 2859c9b95a13SFlorian Westphal release_sock(sk); 2860c9b95a13SFlorian Westphal return -EINVAL; 2861c9b95a13SFlorian Westphal } 2862c9b95a13SFlorian Westphal 2863c9b95a13SFlorian Westphal ret = tcp_setsockopt(ssock->sk, SOL_IPV6, optname, optval, optlen); 2864c9b95a13SFlorian Westphal if (ret == 0) 2865c9b95a13SFlorian Westphal sk->sk_ipv6only = ssock->sk->sk_ipv6only; 2866c9b95a13SFlorian Westphal 2867c9b95a13SFlorian Westphal release_sock(sk); 2868c9b95a13SFlorian Westphal break; 2869c9b95a13SFlorian Westphal } 2870c9b95a13SFlorian Westphal 2871c9b95a13SFlorian Westphal return ret; 2872c9b95a13SFlorian Westphal } 2873c9b95a13SFlorian Westphal 2874717e79c8SPeter Krystad static int mptcp_setsockopt(struct sock *sk, int level, int optname, 2875a7b75c5aSChristoph Hellwig sockptr_t optval, unsigned int optlen) 2876717e79c8SPeter Krystad { 2877717e79c8SPeter Krystad struct mptcp_sock *msk = mptcp_sk(sk); 287876660afbSPaolo Abeni struct sock *ssk; 2879717e79c8SPeter Krystad 2880717e79c8SPeter Krystad pr_debug("msk=%p", msk); 2881717e79c8SPeter Krystad 288283f0c10bSFlorian Westphal if (level == SOL_SOCKET) 2883fd1452d8SFlorian Westphal return mptcp_setsockopt_sol_socket(msk, optname, optval, optlen); 288483f0c10bSFlorian Westphal 2885717e79c8SPeter Krystad /* @@ the meaning of setsockopt() when the socket is connected and 2886b6e4a1aeSMat Martineau * there are multiple subflows is not yet defined. It is up to the 2887b6e4a1aeSMat Martineau * MPTCP-level socket to configure the subflows until the subflow 2888b6e4a1aeSMat Martineau * is in TCP fallback, when TCP socket options are passed through 2889b6e4a1aeSMat Martineau * to the one remaining subflow. 2890717e79c8SPeter Krystad */ 2891717e79c8SPeter Krystad lock_sock(sk); 289276660afbSPaolo Abeni ssk = __mptcp_tcp_fallback(msk); 2893e154659bSFlorian Westphal release_sock(sk); 289476660afbSPaolo Abeni if (ssk) 289576660afbSPaolo Abeni return tcp_setsockopt(ssk, level, optname, optval, optlen); 289650e741bbSFlorian Westphal 2897c9b95a13SFlorian Westphal if (level == SOL_IPV6) 2898c9b95a13SFlorian Westphal return mptcp_setsockopt_v6(msk, optname, optval, optlen); 2899c9b95a13SFlorian Westphal 2900b6e4a1aeSMat Martineau return -EOPNOTSUPP; 2901717e79c8SPeter Krystad } 2902717e79c8SPeter Krystad 2903717e79c8SPeter Krystad static int mptcp_getsockopt(struct sock *sk, int level, int optname, 290450e741bbSFlorian Westphal char __user *optval, int __user *option) 2905717e79c8SPeter Krystad { 2906717e79c8SPeter Krystad struct mptcp_sock *msk = mptcp_sk(sk); 290776660afbSPaolo Abeni struct sock *ssk; 2908717e79c8SPeter Krystad 2909717e79c8SPeter Krystad pr_debug("msk=%p", msk); 2910717e79c8SPeter Krystad 2911b6e4a1aeSMat Martineau /* @@ the meaning of setsockopt() when the socket is connected and 2912b6e4a1aeSMat Martineau * there are multiple subflows is not yet defined. It is up to the 2913b6e4a1aeSMat Martineau * MPTCP-level socket to configure the subflows until the subflow 2914b6e4a1aeSMat Martineau * is in TCP fallback, when socket options are passed through 2915b6e4a1aeSMat Martineau * to the one remaining subflow. 2916717e79c8SPeter Krystad */ 2917717e79c8SPeter Krystad lock_sock(sk); 291876660afbSPaolo Abeni ssk = __mptcp_tcp_fallback(msk); 2919e154659bSFlorian Westphal release_sock(sk); 292076660afbSPaolo Abeni if (ssk) 292176660afbSPaolo Abeni return tcp_getsockopt(ssk, level, optname, optval, option); 292250e741bbSFlorian Westphal 2923b6e4a1aeSMat Martineau return -EOPNOTSUPP; 2924717e79c8SPeter Krystad } 2925717e79c8SPeter Krystad 29266e628cd3SPaolo Abeni void __mptcp_data_acked(struct sock *sk) 29276e628cd3SPaolo Abeni { 29286e628cd3SPaolo Abeni if (!sock_owned_by_user(sk)) 29296e628cd3SPaolo Abeni __mptcp_clean_una(sk); 29306e628cd3SPaolo Abeni else 29316e628cd3SPaolo Abeni set_bit(MPTCP_CLEAN_UNA, &mptcp_sk(sk)->flags); 29326e628cd3SPaolo Abeni 29336e628cd3SPaolo Abeni if (mptcp_pending_data_fin_ack(sk)) 29346e628cd3SPaolo Abeni mptcp_schedule_work(sk); 29356e628cd3SPaolo Abeni } 29366e628cd3SPaolo Abeni 2937219d0499SPaolo Abeni void __mptcp_check_push(struct sock *sk, struct sock *ssk) 29386e628cd3SPaolo Abeni { 29396e628cd3SPaolo Abeni if (!mptcp_send_head(sk)) 29406e628cd3SPaolo Abeni return; 29416e628cd3SPaolo Abeni 294240dc9416SPaolo Abeni if (!sock_owned_by_user(sk)) { 2943b19bc294SPaolo Abeni struct sock *xmit_ssk = mptcp_subflow_get_send(mptcp_sk(sk)); 2944b19bc294SPaolo Abeni 2945b19bc294SPaolo Abeni if (xmit_ssk == ssk) 29466e628cd3SPaolo Abeni __mptcp_subflow_push_pending(sk, ssk); 2947b19bc294SPaolo Abeni else if (xmit_ssk) 2948b19bc294SPaolo Abeni mptcp_subflow_delegate(mptcp_subflow_ctx(xmit_ssk)); 294940dc9416SPaolo Abeni } else { 29506e628cd3SPaolo Abeni set_bit(MPTCP_PUSH_PENDING, &mptcp_sk(sk)->flags); 29516e628cd3SPaolo Abeni } 295240dc9416SPaolo Abeni } 29536e628cd3SPaolo Abeni 2954e93da928SPaolo Abeni /* processes deferred events and flush wmem */ 295514c441b5SPaolo Abeni static void mptcp_release_cb(struct sock *sk) 295614c441b5SPaolo Abeni { 2957c2e6048fSPaolo Abeni for (;;) { 29582d6f5a2bSPaolo Abeni unsigned long flags = 0; 29592d6f5a2bSPaolo Abeni 2960c2e6048fSPaolo Abeni if (test_and_clear_bit(MPTCP_PUSH_PENDING, &mptcp_sk(sk)->flags)) 29612e5de7e0SDan Carpenter flags |= BIT(MPTCP_PUSH_PENDING); 29622d6f5a2bSPaolo Abeni if (test_and_clear_bit(MPTCP_RETRANSMIT, &mptcp_sk(sk)->flags)) 29632d6f5a2bSPaolo Abeni flags |= BIT(MPTCP_RETRANSMIT); 2964c2e6048fSPaolo Abeni if (!flags) 2965c2e6048fSPaolo Abeni break; 2966c2e6048fSPaolo Abeni 2967c2e6048fSPaolo Abeni /* the following actions acquire the subflow socket lock 29686e628cd3SPaolo Abeni * 29696e628cd3SPaolo Abeni * 1) can't be invoked in atomic scope 29706e628cd3SPaolo Abeni * 2) must avoid ABBA deadlock with msk socket spinlock: the RX 29716e628cd3SPaolo Abeni * datapath acquires the msk socket spinlock while helding 29726e628cd3SPaolo Abeni * the subflow socket lock 29736e628cd3SPaolo Abeni */ 29746e628cd3SPaolo Abeni 29756e628cd3SPaolo Abeni spin_unlock_bh(&sk->sk_lock.slock); 29762e5de7e0SDan Carpenter if (flags & BIT(MPTCP_PUSH_PENDING)) 2977c2e6048fSPaolo Abeni __mptcp_push_pending(sk, 0); 29782d6f5a2bSPaolo Abeni if (flags & BIT(MPTCP_RETRANSMIT)) 29792d6f5a2bSPaolo Abeni __mptcp_retrans(sk); 2980c2e6048fSPaolo Abeni 2981c2e6048fSPaolo Abeni cond_resched(); 29826e628cd3SPaolo Abeni spin_lock_bh(&sk->sk_lock.slock); 29836e628cd3SPaolo Abeni } 2984c2e6048fSPaolo Abeni 2985c2e6048fSPaolo Abeni if (test_and_clear_bit(MPTCP_CLEAN_UNA, &mptcp_sk(sk)->flags)) 2986417789dfSPaolo Abeni __mptcp_clean_una_wakeup(sk); 298715cc1045SPaolo Abeni if (test_and_clear_bit(MPTCP_ERROR_REPORT, &mptcp_sk(sk)->flags)) 298815cc1045SPaolo Abeni __mptcp_error_report(sk); 29896e628cd3SPaolo Abeni 2990c2e6048fSPaolo Abeni /* push_pending may touch wmem_reserved, ensure we do the cleanup 2991c2e6048fSPaolo Abeni * later 2992c2e6048fSPaolo Abeni */ 2993e93da928SPaolo Abeni __mptcp_update_wmem(sk); 299487952603SPaolo Abeni __mptcp_update_rmem(sk); 299514c441b5SPaolo Abeni } 299614c441b5SPaolo Abeni 2997b19bc294SPaolo Abeni void mptcp_subflow_process_delegated(struct sock *ssk) 2998b19bc294SPaolo Abeni { 2999b19bc294SPaolo Abeni struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk); 3000b19bc294SPaolo Abeni struct sock *sk = subflow->conn; 3001b19bc294SPaolo Abeni 3002b19bc294SPaolo Abeni mptcp_data_lock(sk); 3003b19bc294SPaolo Abeni if (!sock_owned_by_user(sk)) 3004b19bc294SPaolo Abeni __mptcp_subflow_push_pending(sk, ssk); 3005b19bc294SPaolo Abeni else 3006b19bc294SPaolo Abeni set_bit(MPTCP_PUSH_PENDING, &mptcp_sk(sk)->flags); 3007b19bc294SPaolo Abeni mptcp_data_unlock(sk); 3008b19bc294SPaolo Abeni mptcp_subflow_delegated_done(subflow); 3009b19bc294SPaolo Abeni } 3010b19bc294SPaolo Abeni 30112c5ebd00SPaolo Abeni static int mptcp_hash(struct sock *sk) 30122c5ebd00SPaolo Abeni { 30132c5ebd00SPaolo Abeni /* should never be called, 30142c5ebd00SPaolo Abeni * we hash the TCP subflows not the master socket 30152c5ebd00SPaolo Abeni */ 30162c5ebd00SPaolo Abeni WARN_ON_ONCE(1); 30172c5ebd00SPaolo Abeni return 0; 30182c5ebd00SPaolo Abeni } 30192c5ebd00SPaolo Abeni 30202c5ebd00SPaolo Abeni static void mptcp_unhash(struct sock *sk) 30212c5ebd00SPaolo Abeni { 30222c5ebd00SPaolo Abeni /* called from sk_common_release(), but nothing to do here */ 30232c5ebd00SPaolo Abeni } 30242c5ebd00SPaolo Abeni 3025cec37a6eSPeter Krystad static int mptcp_get_port(struct sock *sk, unsigned short snum) 3026f870fa0bSMat Martineau { 3027f870fa0bSMat Martineau struct mptcp_sock *msk = mptcp_sk(sk); 3028cec37a6eSPeter Krystad struct socket *ssock; 3029f870fa0bSMat Martineau 3030cec37a6eSPeter Krystad ssock = __mptcp_nmpc_socket(msk); 3031cec37a6eSPeter Krystad pr_debug("msk=%p, subflow=%p", msk, ssock); 3032cec37a6eSPeter Krystad if (WARN_ON_ONCE(!ssock)) 3033cec37a6eSPeter Krystad return -EINVAL; 3034f870fa0bSMat Martineau 3035cec37a6eSPeter Krystad return inet_csk_get_port(ssock->sk, snum); 3036cec37a6eSPeter Krystad } 3037f870fa0bSMat Martineau 3038cec37a6eSPeter Krystad void mptcp_finish_connect(struct sock *ssk) 3039cec37a6eSPeter Krystad { 3040cec37a6eSPeter Krystad struct mptcp_subflow_context *subflow; 3041cec37a6eSPeter Krystad struct mptcp_sock *msk; 3042cec37a6eSPeter Krystad struct sock *sk; 30436d0060f6SMat Martineau u64 ack_seq; 3044f870fa0bSMat Martineau 3045cec37a6eSPeter Krystad subflow = mptcp_subflow_ctx(ssk); 3046cec37a6eSPeter Krystad sk = subflow->conn; 3047cec37a6eSPeter Krystad msk = mptcp_sk(sk); 3048cec37a6eSPeter Krystad 3049648ef4b8SMat Martineau pr_debug("msk=%p, token=%u", sk, subflow->token); 3050648ef4b8SMat Martineau 30516d0060f6SMat Martineau mptcp_crypto_key_sha(subflow->remote_key, NULL, &ack_seq); 30526d0060f6SMat Martineau ack_seq++; 3053648ef4b8SMat Martineau subflow->map_seq = ack_seq; 3054648ef4b8SMat Martineau subflow->map_subflow_seq = 1; 30556d0060f6SMat Martineau 3056cec37a6eSPeter Krystad /* the socket is not connected yet, no msk/subflow ops can access/race 3057cec37a6eSPeter Krystad * accessing the field below 3058cec37a6eSPeter Krystad */ 3059cec37a6eSPeter Krystad WRITE_ONCE(msk->remote_key, subflow->remote_key); 3060cec37a6eSPeter Krystad WRITE_ONCE(msk->local_key, subflow->local_key); 30616d0060f6SMat Martineau WRITE_ONCE(msk->write_seq, subflow->idsn + 1); 3062eaa2ffabSPaolo Abeni WRITE_ONCE(msk->snd_nxt, msk->write_seq); 30636d0060f6SMat Martineau WRITE_ONCE(msk->ack_seq, ack_seq); 3064fa3fe2b1SFlorian Westphal WRITE_ONCE(msk->rcv_wnd_sent, ack_seq); 3065d22f4988SChristoph Paasch WRITE_ONCE(msk->can_ack, 1); 30667439d687SPaolo Abeni WRITE_ONCE(msk->snd_una, msk->write_seq); 30671b1c7a0eSPeter Krystad 30686c714f1bSFlorian Westphal mptcp_pm_new_connection(msk, ssk, 0); 3069a6b118feSFlorian Westphal 3070a6b118feSFlorian Westphal mptcp_rcv_space_init(msk, ssk); 3071f870fa0bSMat Martineau } 3072f870fa0bSMat Martineau 3073866f26f2SPaolo Abeni void mptcp_sock_graft(struct sock *sk, struct socket *parent) 3074cf7da0d6SPeter Krystad { 3075cf7da0d6SPeter Krystad write_lock_bh(&sk->sk_callback_lock); 3076cf7da0d6SPeter Krystad rcu_assign_pointer(sk->sk_wq, &parent->wq); 3077cf7da0d6SPeter Krystad sk_set_socket(sk, parent); 3078cf7da0d6SPeter Krystad sk->sk_uid = SOCK_INODE(parent)->i_uid; 3079cf7da0d6SPeter Krystad write_unlock_bh(&sk->sk_callback_lock); 3080cf7da0d6SPeter Krystad } 3081cf7da0d6SPeter Krystad 3082e16163b6SPaolo Abeni bool mptcp_finish_join(struct sock *ssk) 3083f296234cSPeter Krystad { 3084e16163b6SPaolo Abeni struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk); 3085f296234cSPeter Krystad struct mptcp_sock *msk = mptcp_sk(subflow->conn); 3086f296234cSPeter Krystad struct sock *parent = (void *)msk; 3087f296234cSPeter Krystad struct socket *parent_sock; 3088ec3edaa7SPeter Krystad bool ret; 3089f296234cSPeter Krystad 3090f296234cSPeter Krystad pr_debug("msk=%p, subflow=%p", msk, subflow); 3091f296234cSPeter Krystad 3092f296234cSPeter Krystad /* mptcp socket already closing? */ 3093b93df08cSPaolo Abeni if (!mptcp_is_fully_established(parent)) 3094f296234cSPeter Krystad return false; 3095f296234cSPeter Krystad 3096f296234cSPeter Krystad if (!msk->pm.server_side) 3097b911c97cSFlorian Westphal goto out; 3098f296234cSPeter Krystad 309910f6d46cSPaolo Abeni if (!mptcp_pm_allow_new_subflow(msk)) 310010f6d46cSPaolo Abeni return false; 310110f6d46cSPaolo Abeni 310210f6d46cSPaolo Abeni /* active connections are already on conn_list, and we can't acquire 310310f6d46cSPaolo Abeni * msk lock here. 310410f6d46cSPaolo Abeni * use the join list lock as synchronization point and double-check 3105e16163b6SPaolo Abeni * msk status to avoid racing with __mptcp_destroy_sock() 310610f6d46cSPaolo Abeni */ 310710f6d46cSPaolo Abeni spin_lock_bh(&msk->join_list_lock); 310810f6d46cSPaolo Abeni ret = inet_sk_state_load(parent) == TCP_ESTABLISHED; 3109e16163b6SPaolo Abeni if (ret && !WARN_ON_ONCE(!list_empty(&subflow->node))) { 311010f6d46cSPaolo Abeni list_add_tail(&subflow->node, &msk->join_list); 3111e16163b6SPaolo Abeni sock_hold(ssk); 3112e16163b6SPaolo Abeni } 311310f6d46cSPaolo Abeni spin_unlock_bh(&msk->join_list_lock); 311410f6d46cSPaolo Abeni if (!ret) 311510f6d46cSPaolo Abeni return false; 311610f6d46cSPaolo Abeni 311710f6d46cSPaolo Abeni /* attach to msk socket only after we are sure he will deal with us 311810f6d46cSPaolo Abeni * at close time 311910f6d46cSPaolo Abeni */ 3120f296234cSPeter Krystad parent_sock = READ_ONCE(parent->sk_socket); 3121e16163b6SPaolo Abeni if (parent_sock && !ssk->sk_socket) 3122e16163b6SPaolo Abeni mptcp_sock_graft(ssk, parent_sock); 3123917944daSMat Martineau subflow->map_seq = READ_ONCE(msk->ack_seq); 3124b911c97cSFlorian Westphal out: 3125b911c97cSFlorian Westphal mptcp_event(MPTCP_EVENT_SUB_ESTABLISHED, msk, ssk, GFP_ATOMIC); 312610f6d46cSPaolo Abeni return true; 3127f296234cSPeter Krystad } 3128f296234cSPeter Krystad 312976e2a55dSPaolo Abeni static void mptcp_shutdown(struct sock *sk, int how) 313076e2a55dSPaolo Abeni { 313176e2a55dSPaolo Abeni pr_debug("sk=%p, how=%d", sk, how); 313276e2a55dSPaolo Abeni 313376e2a55dSPaolo Abeni if ((how & SEND_SHUTDOWN) && mptcp_close_state(sk)) 313476e2a55dSPaolo Abeni __mptcp_wr_shutdown(sk); 313576e2a55dSPaolo Abeni } 313676e2a55dSPaolo Abeni 3137f870fa0bSMat Martineau static struct proto mptcp_prot = { 3138f870fa0bSMat Martineau .name = "MPTCP", 3139f870fa0bSMat Martineau .owner = THIS_MODULE, 3140f870fa0bSMat Martineau .init = mptcp_init_sock, 314118b683bfSPaolo Abeni .disconnect = mptcp_disconnect, 3142f870fa0bSMat Martineau .close = mptcp_close, 3143cf7da0d6SPeter Krystad .accept = mptcp_accept, 3144717e79c8SPeter Krystad .setsockopt = mptcp_setsockopt, 3145717e79c8SPeter Krystad .getsockopt = mptcp_getsockopt, 314676e2a55dSPaolo Abeni .shutdown = mptcp_shutdown, 314779c0949eSPeter Krystad .destroy = mptcp_destroy, 3148f870fa0bSMat Martineau .sendmsg = mptcp_sendmsg, 3149f870fa0bSMat Martineau .recvmsg = mptcp_recvmsg, 315014c441b5SPaolo Abeni .release_cb = mptcp_release_cb, 31512c5ebd00SPaolo Abeni .hash = mptcp_hash, 31522c5ebd00SPaolo Abeni .unhash = mptcp_unhash, 3153cec37a6eSPeter Krystad .get_port = mptcp_get_port, 3154d027236cSPaolo Abeni .sockets_allocated = &mptcp_sockets_allocated, 3155d027236cSPaolo Abeni .memory_allocated = &tcp_memory_allocated, 3156d027236cSPaolo Abeni .memory_pressure = &tcp_memory_pressure, 3157d027236cSPaolo Abeni .sysctl_wmem_offset = offsetof(struct net, ipv4.sysctl_tcp_wmem), 3158989ef49bSPaolo Abeni .sysctl_rmem_offset = offsetof(struct net, ipv4.sysctl_tcp_rmem), 3159d027236cSPaolo Abeni .sysctl_mem = sysctl_tcp_mem, 3160f870fa0bSMat Martineau .obj_size = sizeof(struct mptcp_sock), 31612c5ebd00SPaolo Abeni .slab_flags = SLAB_TYPESAFE_BY_RCU, 3162f870fa0bSMat Martineau .no_autobind = true, 3163f870fa0bSMat Martineau }; 3164f870fa0bSMat Martineau 31652303f994SPeter Krystad static int mptcp_bind(struct socket *sock, struct sockaddr *uaddr, int addr_len) 31662303f994SPeter Krystad { 31672303f994SPeter Krystad struct mptcp_sock *msk = mptcp_sk(sock->sk); 31682303f994SPeter Krystad struct socket *ssock; 3169cf7da0d6SPeter Krystad int err; 31702303f994SPeter Krystad 31712303f994SPeter Krystad lock_sock(sock->sk); 3172fa68018dSPaolo Abeni ssock = __mptcp_nmpc_socket(msk); 3173fa68018dSPaolo Abeni if (!ssock) { 3174fa68018dSPaolo Abeni err = -EINVAL; 31752303f994SPeter Krystad goto unlock; 31762303f994SPeter Krystad } 31772303f994SPeter Krystad 31782303f994SPeter Krystad err = ssock->ops->bind(ssock, uaddr, addr_len); 3179cf7da0d6SPeter Krystad if (!err) 3180cf7da0d6SPeter Krystad mptcp_copy_inaddrs(sock->sk, ssock->sk); 31812303f994SPeter Krystad 31822303f994SPeter Krystad unlock: 31832303f994SPeter Krystad release_sock(sock->sk); 31842303f994SPeter Krystad return err; 31852303f994SPeter Krystad } 31862303f994SPeter Krystad 31870235d075SPaolo Abeni static void mptcp_subflow_early_fallback(struct mptcp_sock *msk, 31880235d075SPaolo Abeni struct mptcp_subflow_context *subflow) 31890235d075SPaolo Abeni { 31900235d075SPaolo Abeni subflow->request_mptcp = 0; 31910235d075SPaolo Abeni __mptcp_do_fallback(msk); 31920235d075SPaolo Abeni } 31930235d075SPaolo Abeni 31942303f994SPeter Krystad static int mptcp_stream_connect(struct socket *sock, struct sockaddr *uaddr, 31952303f994SPeter Krystad int addr_len, int flags) 31962303f994SPeter Krystad { 31972303f994SPeter Krystad struct mptcp_sock *msk = mptcp_sk(sock->sk); 31982c5ebd00SPaolo Abeni struct mptcp_subflow_context *subflow; 31992303f994SPeter Krystad struct socket *ssock; 32002303f994SPeter Krystad int err; 32012303f994SPeter Krystad 32022303f994SPeter Krystad lock_sock(sock->sk); 320341be81a8SPaolo Abeni if (sock->state != SS_UNCONNECTED && msk->subflow) { 320441be81a8SPaolo Abeni /* pending connection or invalid state, let existing subflow 320541be81a8SPaolo Abeni * cope with that 320641be81a8SPaolo Abeni */ 320741be81a8SPaolo Abeni ssock = msk->subflow; 320841be81a8SPaolo Abeni goto do_connect; 320941be81a8SPaolo Abeni } 321041be81a8SPaolo Abeni 3211fa68018dSPaolo Abeni ssock = __mptcp_nmpc_socket(msk); 3212fa68018dSPaolo Abeni if (!ssock) { 3213fa68018dSPaolo Abeni err = -EINVAL; 32142303f994SPeter Krystad goto unlock; 32152303f994SPeter Krystad } 32162303f994SPeter Krystad 3217fa68018dSPaolo Abeni mptcp_token_destroy(msk); 3218fa68018dSPaolo Abeni inet_sk_state_store(sock->sk, TCP_SYN_SENT); 32192c5ebd00SPaolo Abeni subflow = mptcp_subflow_ctx(ssock->sk); 3220cf7da0d6SPeter Krystad #ifdef CONFIG_TCP_MD5SIG 3221cf7da0d6SPeter Krystad /* no MPTCP if MD5SIG is enabled on this socket or we may run out of 3222cf7da0d6SPeter Krystad * TCP option space. 3223cf7da0d6SPeter Krystad */ 3224cf7da0d6SPeter Krystad if (rcu_access_pointer(tcp_sk(ssock->sk)->md5sig_info)) 32250235d075SPaolo Abeni mptcp_subflow_early_fallback(msk, subflow); 3226cf7da0d6SPeter Krystad #endif 3227a16195e3SPaolo Abeni if (subflow->request_mptcp && mptcp_token_new_connect(ssock->sk)) { 3228a16195e3SPaolo Abeni MPTCP_INC_STATS(sock_net(ssock->sk), MPTCP_MIB_TOKENFALLBACKINIT); 32290235d075SPaolo Abeni mptcp_subflow_early_fallback(msk, subflow); 3230a16195e3SPaolo Abeni } 32315695eb88SPaolo Abeni if (likely(!__mptcp_check_fallback(msk))) 32325695eb88SPaolo Abeni MPTCP_INC_STATS(sock_net(sock->sk), MPTCP_MIB_MPCAPABLEACTIVE); 3233cf7da0d6SPeter Krystad 323441be81a8SPaolo Abeni do_connect: 32352303f994SPeter Krystad err = ssock->ops->connect(ssock, uaddr, addr_len, flags); 323641be81a8SPaolo Abeni sock->state = ssock->state; 323741be81a8SPaolo Abeni 323841be81a8SPaolo Abeni /* on successful connect, the msk state will be moved to established by 323941be81a8SPaolo Abeni * subflow_finish_connect() 324041be81a8SPaolo Abeni */ 3241367fe04eSMatthieu Baerts if (!err || err == -EINPROGRESS) 3242cf7da0d6SPeter Krystad mptcp_copy_inaddrs(sock->sk, ssock->sk); 324341be81a8SPaolo Abeni else 324441be81a8SPaolo Abeni inet_sk_state_store(sock->sk, inet_sk_state_load(ssock->sk)); 32452303f994SPeter Krystad 32462303f994SPeter Krystad unlock: 32472303f994SPeter Krystad release_sock(sock->sk); 32482303f994SPeter Krystad return err; 32492303f994SPeter Krystad } 32502303f994SPeter Krystad 3251cf7da0d6SPeter Krystad static int mptcp_listen(struct socket *sock, int backlog) 3252cf7da0d6SPeter Krystad { 3253cf7da0d6SPeter Krystad struct mptcp_sock *msk = mptcp_sk(sock->sk); 3254cf7da0d6SPeter Krystad struct socket *ssock; 3255cf7da0d6SPeter Krystad int err; 3256cf7da0d6SPeter Krystad 3257cf7da0d6SPeter Krystad pr_debug("msk=%p", msk); 3258cf7da0d6SPeter Krystad 3259cf7da0d6SPeter Krystad lock_sock(sock->sk); 3260fa68018dSPaolo Abeni ssock = __mptcp_nmpc_socket(msk); 3261fa68018dSPaolo Abeni if (!ssock) { 3262fa68018dSPaolo Abeni err = -EINVAL; 3263cf7da0d6SPeter Krystad goto unlock; 3264cf7da0d6SPeter Krystad } 3265cf7da0d6SPeter Krystad 3266fa68018dSPaolo Abeni mptcp_token_destroy(msk); 3267fa68018dSPaolo Abeni inet_sk_state_store(sock->sk, TCP_LISTEN); 32685e20087dSFlorian Westphal sock_set_flag(sock->sk, SOCK_RCU_FREE); 32695e20087dSFlorian Westphal 3270cf7da0d6SPeter Krystad err = ssock->ops->listen(ssock, backlog); 3271cf7da0d6SPeter Krystad inet_sk_state_store(sock->sk, inet_sk_state_load(ssock->sk)); 3272cf7da0d6SPeter Krystad if (!err) 3273cf7da0d6SPeter Krystad mptcp_copy_inaddrs(sock->sk, ssock->sk); 3274cf7da0d6SPeter Krystad 3275cf7da0d6SPeter Krystad unlock: 3276cf7da0d6SPeter Krystad release_sock(sock->sk); 3277cf7da0d6SPeter Krystad return err; 3278cf7da0d6SPeter Krystad } 3279cf7da0d6SPeter Krystad 3280cf7da0d6SPeter Krystad static int mptcp_stream_accept(struct socket *sock, struct socket *newsock, 3281cf7da0d6SPeter Krystad int flags, bool kern) 3282cf7da0d6SPeter Krystad { 3283cf7da0d6SPeter Krystad struct mptcp_sock *msk = mptcp_sk(sock->sk); 3284cf7da0d6SPeter Krystad struct socket *ssock; 3285cf7da0d6SPeter Krystad int err; 3286cf7da0d6SPeter Krystad 3287cf7da0d6SPeter Krystad pr_debug("msk=%p", msk); 3288cf7da0d6SPeter Krystad 3289cf7da0d6SPeter Krystad lock_sock(sock->sk); 3290cf7da0d6SPeter Krystad if (sock->sk->sk_state != TCP_LISTEN) 3291cf7da0d6SPeter Krystad goto unlock_fail; 3292cf7da0d6SPeter Krystad 3293cf7da0d6SPeter Krystad ssock = __mptcp_nmpc_socket(msk); 3294cf7da0d6SPeter Krystad if (!ssock) 3295cf7da0d6SPeter Krystad goto unlock_fail; 3296cf7da0d6SPeter Krystad 32978a05661bSPaolo Abeni clear_bit(MPTCP_DATA_READY, &msk->flags); 3298cf7da0d6SPeter Krystad sock_hold(ssock->sk); 3299cf7da0d6SPeter Krystad release_sock(sock->sk); 3300cf7da0d6SPeter Krystad 3301cf7da0d6SPeter Krystad err = ssock->ops->accept(sock, newsock, flags, kern); 3302d2f77c53SPaolo Abeni if (err == 0 && !mptcp_is_tcpsk(newsock->sk)) { 3303cf7da0d6SPeter Krystad struct mptcp_sock *msk = mptcp_sk(newsock->sk); 3304cf7da0d6SPeter Krystad struct mptcp_subflow_context *subflow; 33050397c6d8SPaolo Abeni struct sock *newsk = newsock->sk; 33060397c6d8SPaolo Abeni 33074d54cc32SFlorian Westphal lock_sock(newsk); 33085b950ff4SPaolo Abeni 33095b950ff4SPaolo Abeni /* PM/worker can now acquire the first subflow socket 33105b950ff4SPaolo Abeni * lock without racing with listener queue cleanup, 33115b950ff4SPaolo Abeni * we can notify it, if needed. 3312c8fe62f0SFlorian Westphal * 3313c8fe62f0SFlorian Westphal * Even if remote has reset the initial subflow by now 3314c8fe62f0SFlorian Westphal * the refcnt is still at least one. 33155b950ff4SPaolo Abeni */ 33165b950ff4SPaolo Abeni subflow = mptcp_subflow_ctx(msk->first); 33175b950ff4SPaolo Abeni list_add(&subflow->node, &msk->conn_list); 33185b950ff4SPaolo Abeni sock_hold(msk->first); 33195b950ff4SPaolo Abeni if (mptcp_is_fully_established(newsk)) 33206c714f1bSFlorian Westphal mptcp_pm_fully_established(msk, msk->first, GFP_KERNEL); 33215b950ff4SPaolo Abeni 33220397c6d8SPaolo Abeni mptcp_copy_inaddrs(newsk, msk->first); 33230397c6d8SPaolo Abeni mptcp_rcv_space_init(msk, msk->first); 33245cf92bbaSPaolo Abeni mptcp_propagate_sndbuf(newsk, msk->first); 3325cf7da0d6SPeter Krystad 3326cf7da0d6SPeter Krystad /* set ssk->sk_socket of accept()ed flows to mptcp socket. 3327cf7da0d6SPeter Krystad * This is needed so NOSPACE flag can be set from tcp stack. 3328cf7da0d6SPeter Krystad */ 3329ec3edaa7SPeter Krystad __mptcp_flush_join_list(msk); 3330190f8b06SGeliang Tang mptcp_for_each_subflow(msk, subflow) { 3331cf7da0d6SPeter Krystad struct sock *ssk = mptcp_subflow_tcp_sock(subflow); 3332cf7da0d6SPeter Krystad 3333cf7da0d6SPeter Krystad if (!ssk->sk_socket) 3334cf7da0d6SPeter Krystad mptcp_sock_graft(ssk, newsock); 3335cf7da0d6SPeter Krystad } 33364d54cc32SFlorian Westphal release_sock(newsk); 3337cf7da0d6SPeter Krystad } 3338cf7da0d6SPeter Krystad 33398a05661bSPaolo Abeni if (inet_csk_listen_poll(ssock->sk)) 33408a05661bSPaolo Abeni set_bit(MPTCP_DATA_READY, &msk->flags); 3341cf7da0d6SPeter Krystad sock_put(ssock->sk); 3342cf7da0d6SPeter Krystad return err; 3343cf7da0d6SPeter Krystad 3344cf7da0d6SPeter Krystad unlock_fail: 3345cf7da0d6SPeter Krystad release_sock(sock->sk); 3346cf7da0d6SPeter Krystad return -EINVAL; 3347cf7da0d6SPeter Krystad } 3348cf7da0d6SPeter Krystad 33498a05661bSPaolo Abeni static __poll_t mptcp_check_readable(struct mptcp_sock *msk) 33508a05661bSPaolo Abeni { 33518a05661bSPaolo Abeni return test_bit(MPTCP_DATA_READY, &msk->flags) ? EPOLLIN | EPOLLRDNORM : 33528a05661bSPaolo Abeni 0; 33538a05661bSPaolo Abeni } 33548a05661bSPaolo Abeni 33558edf0864SFlorian Westphal static __poll_t mptcp_check_writeable(struct mptcp_sock *msk) 33568edf0864SFlorian Westphal { 33578edf0864SFlorian Westphal struct sock *sk = (struct sock *)msk; 33588edf0864SFlorian Westphal 33598edf0864SFlorian Westphal if (unlikely(sk->sk_shutdown & SEND_SHUTDOWN)) 3360dd913410SPaolo Abeni return EPOLLOUT | EPOLLWRNORM; 33618edf0864SFlorian Westphal 33628edf0864SFlorian Westphal if (sk_stream_is_writeable(sk)) 33638edf0864SFlorian Westphal return EPOLLOUT | EPOLLWRNORM; 33648edf0864SFlorian Westphal 33655cf92bbaSPaolo Abeni mptcp_set_nospace(sk); 33666e628cd3SPaolo Abeni smp_mb__after_atomic(); /* msk->flags is changed by write_space cb */ 33676e628cd3SPaolo Abeni if (sk_stream_is_writeable(sk)) 33686e628cd3SPaolo Abeni return EPOLLOUT | EPOLLWRNORM; 33698edf0864SFlorian Westphal 33706e628cd3SPaolo Abeni return 0; 33718edf0864SFlorian Westphal } 33728edf0864SFlorian Westphal 33732303f994SPeter Krystad static __poll_t mptcp_poll(struct file *file, struct socket *sock, 33742303f994SPeter Krystad struct poll_table_struct *wait) 33752303f994SPeter Krystad { 33761891c4a0SFlorian Westphal struct sock *sk = sock->sk; 33778ab183deSPaolo Abeni struct mptcp_sock *msk; 33782303f994SPeter Krystad __poll_t mask = 0; 33798a05661bSPaolo Abeni int state; 33802303f994SPeter Krystad 33811891c4a0SFlorian Westphal msk = mptcp_sk(sk); 33821891c4a0SFlorian Westphal sock_poll_wait(file, sock, wait); 33831891c4a0SFlorian Westphal 33848a05661bSPaolo Abeni state = inet_sk_state_load(sk); 33856719331cSPaolo Abeni pr_debug("msk=%p state=%d flags=%lx", msk, state, msk->flags); 33868a05661bSPaolo Abeni if (state == TCP_LISTEN) 33878a05661bSPaolo Abeni return mptcp_check_readable(msk); 33888a05661bSPaolo Abeni 33898a05661bSPaolo Abeni if (state != TCP_SYN_SENT && state != TCP_SYN_RECV) { 33908a05661bSPaolo Abeni mask |= mptcp_check_readable(msk); 33918edf0864SFlorian Westphal mask |= mptcp_check_writeable(msk); 33928a05661bSPaolo Abeni } 3393dd913410SPaolo Abeni if (sk->sk_shutdown == SHUTDOWN_MASK || state == TCP_CLOSE) 3394dd913410SPaolo Abeni mask |= EPOLLHUP; 33951891c4a0SFlorian Westphal if (sk->sk_shutdown & RCV_SHUTDOWN) 33961891c4a0SFlorian Westphal mask |= EPOLLIN | EPOLLRDNORM | EPOLLRDHUP; 33971891c4a0SFlorian Westphal 339815cc1045SPaolo Abeni /* This barrier is coupled with smp_wmb() in tcp_reset() */ 339915cc1045SPaolo Abeni smp_rmb(); 340015cc1045SPaolo Abeni if (sk->sk_err) 340115cc1045SPaolo Abeni mask |= EPOLLERR; 340215cc1045SPaolo Abeni 34032303f994SPeter Krystad return mask; 34042303f994SPeter Krystad } 34052303f994SPeter Krystad 3406ad98dd37SFlorian Westphal static int mptcp_release(struct socket *sock) 3407ad98dd37SFlorian Westphal { 3408ad98dd37SFlorian Westphal struct mptcp_subflow_context *subflow; 3409ad98dd37SFlorian Westphal struct sock *sk = sock->sk; 3410ad98dd37SFlorian Westphal struct mptcp_sock *msk; 3411ad98dd37SFlorian Westphal 3412ad98dd37SFlorian Westphal if (!sk) 3413ad98dd37SFlorian Westphal return 0; 3414ad98dd37SFlorian Westphal 3415ad98dd37SFlorian Westphal lock_sock(sk); 3416ad98dd37SFlorian Westphal 3417ad98dd37SFlorian Westphal msk = mptcp_sk(sk); 3418ad98dd37SFlorian Westphal 3419ad98dd37SFlorian Westphal mptcp_for_each_subflow(msk, subflow) { 3420ad98dd37SFlorian Westphal struct sock *ssk = mptcp_subflow_tcp_sock(subflow); 3421ad98dd37SFlorian Westphal 3422ad98dd37SFlorian Westphal ip_mc_drop_socket(ssk); 3423ad98dd37SFlorian Westphal } 3424ad98dd37SFlorian Westphal 3425ad98dd37SFlorian Westphal release_sock(sk); 3426ad98dd37SFlorian Westphal 3427ad98dd37SFlorian Westphal return inet_release(sock); 3428ad98dd37SFlorian Westphal } 3429ad98dd37SFlorian Westphal 3430e42f1ac6SFlorian Westphal static const struct proto_ops mptcp_stream_ops = { 3431e42f1ac6SFlorian Westphal .family = PF_INET, 3432e42f1ac6SFlorian Westphal .owner = THIS_MODULE, 3433ad98dd37SFlorian Westphal .release = mptcp_release, 3434e42f1ac6SFlorian Westphal .bind = mptcp_bind, 3435e42f1ac6SFlorian Westphal .connect = mptcp_stream_connect, 3436e42f1ac6SFlorian Westphal .socketpair = sock_no_socketpair, 3437e42f1ac6SFlorian Westphal .accept = mptcp_stream_accept, 3438d2f77c53SPaolo Abeni .getname = inet_getname, 3439e42f1ac6SFlorian Westphal .poll = mptcp_poll, 3440e42f1ac6SFlorian Westphal .ioctl = inet_ioctl, 3441e42f1ac6SFlorian Westphal .gettstamp = sock_gettstamp, 3442e42f1ac6SFlorian Westphal .listen = mptcp_listen, 344376e2a55dSPaolo Abeni .shutdown = inet_shutdown, 3444e42f1ac6SFlorian Westphal .setsockopt = sock_common_setsockopt, 3445e42f1ac6SFlorian Westphal .getsockopt = sock_common_getsockopt, 3446e42f1ac6SFlorian Westphal .sendmsg = inet_sendmsg, 3447e42f1ac6SFlorian Westphal .recvmsg = inet_recvmsg, 3448e42f1ac6SFlorian Westphal .mmap = sock_no_mmap, 3449e42f1ac6SFlorian Westphal .sendpage = inet_sendpage, 3450e42f1ac6SFlorian Westphal }; 34512303f994SPeter Krystad 3452f870fa0bSMat Martineau static struct inet_protosw mptcp_protosw = { 3453f870fa0bSMat Martineau .type = SOCK_STREAM, 3454f870fa0bSMat Martineau .protocol = IPPROTO_MPTCP, 3455f870fa0bSMat Martineau .prot = &mptcp_prot, 34562303f994SPeter Krystad .ops = &mptcp_stream_ops, 34572303f994SPeter Krystad .flags = INET_PROTOSW_ICSK, 3458f870fa0bSMat Martineau }; 3459f870fa0bSMat Martineau 3460b19bc294SPaolo Abeni static int mptcp_napi_poll(struct napi_struct *napi, int budget) 3461b19bc294SPaolo Abeni { 3462b19bc294SPaolo Abeni struct mptcp_delegated_action *delegated; 3463b19bc294SPaolo Abeni struct mptcp_subflow_context *subflow; 3464b19bc294SPaolo Abeni int work_done = 0; 3465b19bc294SPaolo Abeni 3466b19bc294SPaolo Abeni delegated = container_of(napi, struct mptcp_delegated_action, napi); 3467b19bc294SPaolo Abeni while ((subflow = mptcp_subflow_delegated_next(delegated)) != NULL) { 3468b19bc294SPaolo Abeni struct sock *ssk = mptcp_subflow_tcp_sock(subflow); 3469b19bc294SPaolo Abeni 3470b19bc294SPaolo Abeni bh_lock_sock_nested(ssk); 3471b19bc294SPaolo Abeni if (!sock_owned_by_user(ssk) && 3472b19bc294SPaolo Abeni mptcp_subflow_has_delegated_action(subflow)) 3473b19bc294SPaolo Abeni mptcp_subflow_process_delegated(ssk); 3474b19bc294SPaolo Abeni /* ... elsewhere tcp_release_cb_override already processed 3475b19bc294SPaolo Abeni * the action or will do at next release_sock(). 3476b19bc294SPaolo Abeni * In both case must dequeue the subflow here - on the same 3477b19bc294SPaolo Abeni * CPU that scheduled it. 3478b19bc294SPaolo Abeni */ 3479b19bc294SPaolo Abeni bh_unlock_sock(ssk); 3480b19bc294SPaolo Abeni sock_put(ssk); 3481b19bc294SPaolo Abeni 3482b19bc294SPaolo Abeni if (++work_done == budget) 3483b19bc294SPaolo Abeni return budget; 3484b19bc294SPaolo Abeni } 3485b19bc294SPaolo Abeni 3486b19bc294SPaolo Abeni /* always provide a 0 'work_done' argument, so that napi_complete_done 3487b19bc294SPaolo Abeni * will not try accessing the NULL napi->dev ptr 3488b19bc294SPaolo Abeni */ 3489b19bc294SPaolo Abeni napi_complete_done(napi, 0); 3490b19bc294SPaolo Abeni return work_done; 3491b19bc294SPaolo Abeni } 3492b19bc294SPaolo Abeni 3493d39dcecaSPaolo Abeni void __init mptcp_proto_init(void) 3494f870fa0bSMat Martineau { 3495b19bc294SPaolo Abeni struct mptcp_delegated_action *delegated; 3496b19bc294SPaolo Abeni int cpu; 3497b19bc294SPaolo Abeni 34982303f994SPeter Krystad mptcp_prot.h.hashinfo = tcp_prot.h.hashinfo; 34992303f994SPeter Krystad 3500d027236cSPaolo Abeni if (percpu_counter_init(&mptcp_sockets_allocated, 0, GFP_KERNEL)) 3501d027236cSPaolo Abeni panic("Failed to allocate MPTCP pcpu counter\n"); 3502d027236cSPaolo Abeni 3503b19bc294SPaolo Abeni init_dummy_netdev(&mptcp_napi_dev); 3504b19bc294SPaolo Abeni for_each_possible_cpu(cpu) { 3505b19bc294SPaolo Abeni delegated = per_cpu_ptr(&mptcp_delegated_actions, cpu); 3506b19bc294SPaolo Abeni INIT_LIST_HEAD(&delegated->head); 3507b19bc294SPaolo Abeni netif_tx_napi_add(&mptcp_napi_dev, &delegated->napi, mptcp_napi_poll, 3508b19bc294SPaolo Abeni NAPI_POLL_WEIGHT); 3509b19bc294SPaolo Abeni napi_enable(&delegated->napi); 3510b19bc294SPaolo Abeni } 3511b19bc294SPaolo Abeni 35122303f994SPeter Krystad mptcp_subflow_init(); 35131b1c7a0eSPeter Krystad mptcp_pm_init(); 35142c5ebd00SPaolo Abeni mptcp_token_init(); 35152303f994SPeter Krystad 3516f870fa0bSMat Martineau if (proto_register(&mptcp_prot, 1) != 0) 3517f870fa0bSMat Martineau panic("Failed to register MPTCP proto.\n"); 3518f870fa0bSMat Martineau 3519f870fa0bSMat Martineau inet_register_protosw(&mptcp_protosw); 35206771bfd9SFlorian Westphal 35216771bfd9SFlorian Westphal BUILD_BUG_ON(sizeof(struct mptcp_skb_cb) > sizeof_field(struct sk_buff, cb)); 3522f870fa0bSMat Martineau } 3523f870fa0bSMat Martineau 3524f870fa0bSMat Martineau #if IS_ENABLED(CONFIG_MPTCP_IPV6) 3525ad98dd37SFlorian Westphal static int mptcp6_release(struct socket *sock) 3526ad98dd37SFlorian Westphal { 3527ad98dd37SFlorian Westphal struct mptcp_subflow_context *subflow; 3528ad98dd37SFlorian Westphal struct mptcp_sock *msk; 3529ad98dd37SFlorian Westphal struct sock *sk = sock->sk; 3530ad98dd37SFlorian Westphal 3531ad98dd37SFlorian Westphal if (!sk) 3532ad98dd37SFlorian Westphal return 0; 3533ad98dd37SFlorian Westphal 3534ad98dd37SFlorian Westphal lock_sock(sk); 3535ad98dd37SFlorian Westphal 3536ad98dd37SFlorian Westphal msk = mptcp_sk(sk); 3537ad98dd37SFlorian Westphal 3538ad98dd37SFlorian Westphal mptcp_for_each_subflow(msk, subflow) { 3539ad98dd37SFlorian Westphal struct sock *ssk = mptcp_subflow_tcp_sock(subflow); 3540ad98dd37SFlorian Westphal 3541ad98dd37SFlorian Westphal ip_mc_drop_socket(ssk); 3542ad98dd37SFlorian Westphal ipv6_sock_mc_close(ssk); 3543ad98dd37SFlorian Westphal ipv6_sock_ac_close(ssk); 3544ad98dd37SFlorian Westphal } 3545ad98dd37SFlorian Westphal 3546ad98dd37SFlorian Westphal release_sock(sk); 3547ad98dd37SFlorian Westphal return inet6_release(sock); 3548ad98dd37SFlorian Westphal } 3549ad98dd37SFlorian Westphal 3550e42f1ac6SFlorian Westphal static const struct proto_ops mptcp_v6_stream_ops = { 3551e42f1ac6SFlorian Westphal .family = PF_INET6, 3552e42f1ac6SFlorian Westphal .owner = THIS_MODULE, 3553ad98dd37SFlorian Westphal .release = mptcp6_release, 3554e42f1ac6SFlorian Westphal .bind = mptcp_bind, 3555e42f1ac6SFlorian Westphal .connect = mptcp_stream_connect, 3556e42f1ac6SFlorian Westphal .socketpair = sock_no_socketpair, 3557e42f1ac6SFlorian Westphal .accept = mptcp_stream_accept, 3558d2f77c53SPaolo Abeni .getname = inet6_getname, 3559e42f1ac6SFlorian Westphal .poll = mptcp_poll, 3560e42f1ac6SFlorian Westphal .ioctl = inet6_ioctl, 3561e42f1ac6SFlorian Westphal .gettstamp = sock_gettstamp, 3562e42f1ac6SFlorian Westphal .listen = mptcp_listen, 356376e2a55dSPaolo Abeni .shutdown = inet_shutdown, 3564e42f1ac6SFlorian Westphal .setsockopt = sock_common_setsockopt, 3565e42f1ac6SFlorian Westphal .getsockopt = sock_common_getsockopt, 3566e42f1ac6SFlorian Westphal .sendmsg = inet6_sendmsg, 3567e42f1ac6SFlorian Westphal .recvmsg = inet6_recvmsg, 3568e42f1ac6SFlorian Westphal .mmap = sock_no_mmap, 3569e42f1ac6SFlorian Westphal .sendpage = inet_sendpage, 3570e42f1ac6SFlorian Westphal #ifdef CONFIG_COMPAT 35713986912fSChristoph Hellwig .compat_ioctl = inet6_compat_ioctl, 3572e42f1ac6SFlorian Westphal #endif 3573e42f1ac6SFlorian Westphal }; 3574e42f1ac6SFlorian Westphal 3575f870fa0bSMat Martineau static struct proto mptcp_v6_prot; 3576f870fa0bSMat Martineau 357779c0949eSPeter Krystad static void mptcp_v6_destroy(struct sock *sk) 357879c0949eSPeter Krystad { 357979c0949eSPeter Krystad mptcp_destroy(sk); 358079c0949eSPeter Krystad inet6_destroy_sock(sk); 358179c0949eSPeter Krystad } 358279c0949eSPeter Krystad 3583f870fa0bSMat Martineau static struct inet_protosw mptcp_v6_protosw = { 3584f870fa0bSMat Martineau .type = SOCK_STREAM, 3585f870fa0bSMat Martineau .protocol = IPPROTO_MPTCP, 3586f870fa0bSMat Martineau .prot = &mptcp_v6_prot, 35872303f994SPeter Krystad .ops = &mptcp_v6_stream_ops, 3588f870fa0bSMat Martineau .flags = INET_PROTOSW_ICSK, 3589f870fa0bSMat Martineau }; 3590f870fa0bSMat Martineau 3591d39dcecaSPaolo Abeni int __init mptcp_proto_v6_init(void) 3592f870fa0bSMat Martineau { 3593f870fa0bSMat Martineau int err; 3594f870fa0bSMat Martineau 3595f870fa0bSMat Martineau mptcp_v6_prot = mptcp_prot; 3596f870fa0bSMat Martineau strcpy(mptcp_v6_prot.name, "MPTCPv6"); 3597f870fa0bSMat Martineau mptcp_v6_prot.slab = NULL; 359879c0949eSPeter Krystad mptcp_v6_prot.destroy = mptcp_v6_destroy; 3599b0519de8SFlorian Westphal mptcp_v6_prot.obj_size = sizeof(struct mptcp6_sock); 3600f870fa0bSMat Martineau 3601f870fa0bSMat Martineau err = proto_register(&mptcp_v6_prot, 1); 3602f870fa0bSMat Martineau if (err) 3603f870fa0bSMat Martineau return err; 3604f870fa0bSMat Martineau 3605f870fa0bSMat Martineau err = inet6_register_protosw(&mptcp_v6_protosw); 3606f870fa0bSMat Martineau if (err) 3607f870fa0bSMat Martineau proto_unregister(&mptcp_v6_prot); 3608f870fa0bSMat Martineau 3609f870fa0bSMat Martineau return err; 3610f870fa0bSMat Martineau } 3611f870fa0bSMat Martineau #endif 3612