1 // SPDX-License-Identifier: GPL-2.0 2 /* Multipath TCP 3 * 4 * Copyright (c) 2017 - 2019, Intel Corporation. 5 */ 6 7 #define pr_fmt(fmt) "MPTCP: " fmt 8 9 #include <linux/kernel.h> 10 #include <linux/module.h> 11 #include <linux/netdevice.h> 12 #include <linux/sched/signal.h> 13 #include <linux/atomic.h> 14 #include <net/sock.h> 15 #include <net/inet_common.h> 16 #include <net/inet_hashtables.h> 17 #include <net/protocol.h> 18 #include <net/tcp.h> 19 #include <net/tcp_states.h> 20 #if IS_ENABLED(CONFIG_MPTCP_IPV6) 21 #include <net/transp_v6.h> 22 #endif 23 #include <net/mptcp.h> 24 #include <net/xfrm.h> 25 #include "protocol.h" 26 #include "mib.h" 27 28 #define CREATE_TRACE_POINTS 29 #include <trace/events/mptcp.h> 30 31 #if IS_ENABLED(CONFIG_MPTCP_IPV6) 32 struct mptcp6_sock { 33 struct mptcp_sock msk; 34 struct ipv6_pinfo np; 35 }; 36 #endif 37 38 struct mptcp_skb_cb { 39 u64 map_seq; 40 u64 end_seq; 41 u32 offset; 42 u8 has_rxtstamp:1; 43 }; 44 45 #define MPTCP_SKB_CB(__skb) ((struct mptcp_skb_cb *)&((__skb)->cb[0])) 46 47 enum { 48 MPTCP_CMSG_TS = BIT(0), 49 }; 50 51 static struct percpu_counter mptcp_sockets_allocated; 52 53 static void __mptcp_destroy_sock(struct sock *sk); 54 static void __mptcp_check_send_data_fin(struct sock *sk); 55 56 DEFINE_PER_CPU(struct mptcp_delegated_action, mptcp_delegated_actions); 57 static struct net_device mptcp_napi_dev; 58 59 /* If msk has an initial subflow socket, and the MP_CAPABLE handshake has not 60 * completed yet or has failed, return the subflow socket. 61 * Otherwise return NULL. 62 */ 63 struct socket *__mptcp_nmpc_socket(const struct mptcp_sock *msk) 64 { 65 if (!msk->subflow || READ_ONCE(msk->can_ack)) 66 return NULL; 67 68 return msk->subflow; 69 } 70 71 /* Returns end sequence number of the receiver's advertised window */ 72 static u64 mptcp_wnd_end(const struct mptcp_sock *msk) 73 { 74 return READ_ONCE(msk->wnd_end); 75 } 76 77 static bool mptcp_is_tcpsk(struct sock *sk) 78 { 79 struct socket *sock = sk->sk_socket; 80 81 if (unlikely(sk->sk_prot == &tcp_prot)) { 82 /* we are being invoked after mptcp_accept() has 83 * accepted a non-mp-capable flow: sk is a tcp_sk, 84 * not an mptcp one. 85 * 86 * Hand the socket over to tcp so all further socket ops 87 * bypass mptcp. 88 */ 89 sock->ops = &inet_stream_ops; 90 return true; 91 #if IS_ENABLED(CONFIG_MPTCP_IPV6) 92 } else if (unlikely(sk->sk_prot == &tcpv6_prot)) { 93 sock->ops = &inet6_stream_ops; 94 return true; 95 #endif 96 } 97 98 return false; 99 } 100 101 static int __mptcp_socket_create(struct mptcp_sock *msk) 102 { 103 struct mptcp_subflow_context *subflow; 104 struct sock *sk = (struct sock *)msk; 105 struct socket *ssock; 106 int err; 107 108 err = mptcp_subflow_create_socket(sk, &ssock); 109 if (err) 110 return err; 111 112 msk->first = ssock->sk; 113 msk->subflow = ssock; 114 subflow = mptcp_subflow_ctx(ssock->sk); 115 list_add(&subflow->node, &msk->conn_list); 116 sock_hold(ssock->sk); 117 subflow->request_mptcp = 1; 118 mptcp_sock_graft(msk->first, sk->sk_socket); 119 120 return 0; 121 } 122 123 static void mptcp_drop(struct sock *sk, struct sk_buff *skb) 124 { 125 sk_drops_add(sk, skb); 126 __kfree_skb(skb); 127 } 128 129 static bool mptcp_try_coalesce(struct sock *sk, struct sk_buff *to, 130 struct sk_buff *from) 131 { 132 bool fragstolen; 133 int delta; 134 135 if (MPTCP_SKB_CB(from)->offset || 136 !skb_try_coalesce(to, from, &fragstolen, &delta)) 137 return false; 138 139 pr_debug("colesced seq %llx into %llx new len %d new end seq %llx", 140 MPTCP_SKB_CB(from)->map_seq, MPTCP_SKB_CB(to)->map_seq, 141 to->len, MPTCP_SKB_CB(from)->end_seq); 142 MPTCP_SKB_CB(to)->end_seq = MPTCP_SKB_CB(from)->end_seq; 143 kfree_skb_partial(from, fragstolen); 144 atomic_add(delta, &sk->sk_rmem_alloc); 145 sk_mem_charge(sk, delta); 146 return true; 147 } 148 149 static bool mptcp_ooo_try_coalesce(struct mptcp_sock *msk, struct sk_buff *to, 150 struct sk_buff *from) 151 { 152 if (MPTCP_SKB_CB(from)->map_seq != MPTCP_SKB_CB(to)->end_seq) 153 return false; 154 155 return mptcp_try_coalesce((struct sock *)msk, to, from); 156 } 157 158 /* "inspired" by tcp_data_queue_ofo(), main differences: 159 * - use mptcp seqs 160 * - don't cope with sacks 161 */ 162 static void mptcp_data_queue_ofo(struct mptcp_sock *msk, struct sk_buff *skb) 163 { 164 struct sock *sk = (struct sock *)msk; 165 struct rb_node **p, *parent; 166 u64 seq, end_seq, max_seq; 167 struct sk_buff *skb1; 168 169 seq = MPTCP_SKB_CB(skb)->map_seq; 170 end_seq = MPTCP_SKB_CB(skb)->end_seq; 171 max_seq = READ_ONCE(msk->rcv_wnd_sent); 172 173 pr_debug("msk=%p seq=%llx limit=%llx empty=%d", msk, seq, max_seq, 174 RB_EMPTY_ROOT(&msk->out_of_order_queue)); 175 if (after64(end_seq, max_seq)) { 176 /* out of window */ 177 mptcp_drop(sk, skb); 178 pr_debug("oow by %lld, rcv_wnd_sent %llu\n", 179 (unsigned long long)end_seq - (unsigned long)max_seq, 180 (unsigned long long)msk->rcv_wnd_sent); 181 MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_NODSSWINDOW); 182 return; 183 } 184 185 p = &msk->out_of_order_queue.rb_node; 186 MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_OFOQUEUE); 187 if (RB_EMPTY_ROOT(&msk->out_of_order_queue)) { 188 rb_link_node(&skb->rbnode, NULL, p); 189 rb_insert_color(&skb->rbnode, &msk->out_of_order_queue); 190 msk->ooo_last_skb = skb; 191 goto end; 192 } 193 194 /* with 2 subflows, adding at end of ooo queue is quite likely 195 * Use of ooo_last_skb avoids the O(Log(N)) rbtree lookup. 196 */ 197 if (mptcp_ooo_try_coalesce(msk, msk->ooo_last_skb, skb)) { 198 MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_OFOMERGE); 199 MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_OFOQUEUETAIL); 200 return; 201 } 202 203 /* Can avoid an rbtree lookup if we are adding skb after ooo_last_skb */ 204 if (!before64(seq, MPTCP_SKB_CB(msk->ooo_last_skb)->end_seq)) { 205 MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_OFOQUEUETAIL); 206 parent = &msk->ooo_last_skb->rbnode; 207 p = &parent->rb_right; 208 goto insert; 209 } 210 211 /* Find place to insert this segment. Handle overlaps on the way. */ 212 parent = NULL; 213 while (*p) { 214 parent = *p; 215 skb1 = rb_to_skb(parent); 216 if (before64(seq, MPTCP_SKB_CB(skb1)->map_seq)) { 217 p = &parent->rb_left; 218 continue; 219 } 220 if (before64(seq, MPTCP_SKB_CB(skb1)->end_seq)) { 221 if (!after64(end_seq, MPTCP_SKB_CB(skb1)->end_seq)) { 222 /* All the bits are present. Drop. */ 223 mptcp_drop(sk, skb); 224 MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_DUPDATA); 225 return; 226 } 227 if (after64(seq, MPTCP_SKB_CB(skb1)->map_seq)) { 228 /* partial overlap: 229 * | skb | 230 * | skb1 | 231 * continue traversing 232 */ 233 } else { 234 /* skb's seq == skb1's seq and skb covers skb1. 235 * Replace skb1 with skb. 236 */ 237 rb_replace_node(&skb1->rbnode, &skb->rbnode, 238 &msk->out_of_order_queue); 239 mptcp_drop(sk, skb1); 240 MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_DUPDATA); 241 goto merge_right; 242 } 243 } else if (mptcp_ooo_try_coalesce(msk, skb1, skb)) { 244 MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_OFOMERGE); 245 return; 246 } 247 p = &parent->rb_right; 248 } 249 250 insert: 251 /* Insert segment into RB tree. */ 252 rb_link_node(&skb->rbnode, parent, p); 253 rb_insert_color(&skb->rbnode, &msk->out_of_order_queue); 254 255 merge_right: 256 /* Remove other segments covered by skb. */ 257 while ((skb1 = skb_rb_next(skb)) != NULL) { 258 if (before64(end_seq, MPTCP_SKB_CB(skb1)->end_seq)) 259 break; 260 rb_erase(&skb1->rbnode, &msk->out_of_order_queue); 261 mptcp_drop(sk, skb1); 262 MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_DUPDATA); 263 } 264 /* If there is no skb after us, we are the last_skb ! */ 265 if (!skb1) 266 msk->ooo_last_skb = skb; 267 268 end: 269 skb_condense(skb); 270 skb_set_owner_r(skb, sk); 271 } 272 273 static bool __mptcp_move_skb(struct mptcp_sock *msk, struct sock *ssk, 274 struct sk_buff *skb, unsigned int offset, 275 size_t copy_len) 276 { 277 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk); 278 struct sock *sk = (struct sock *)msk; 279 struct sk_buff *tail; 280 bool has_rxtstamp; 281 282 __skb_unlink(skb, &ssk->sk_receive_queue); 283 284 skb_ext_reset(skb); 285 skb_orphan(skb); 286 287 /* try to fetch required memory from subflow */ 288 if (!sk_rmem_schedule(sk, skb, skb->truesize)) { 289 int amount = sk_mem_pages(skb->truesize) << SK_MEM_QUANTUM_SHIFT; 290 291 if (ssk->sk_forward_alloc < amount) 292 goto drop; 293 294 ssk->sk_forward_alloc -= amount; 295 sk->sk_forward_alloc += amount; 296 } 297 298 has_rxtstamp = TCP_SKB_CB(skb)->has_rxtstamp; 299 300 /* the skb map_seq accounts for the skb offset: 301 * mptcp_subflow_get_mapped_dsn() is based on the current tp->copied_seq 302 * value 303 */ 304 MPTCP_SKB_CB(skb)->map_seq = mptcp_subflow_get_mapped_dsn(subflow); 305 MPTCP_SKB_CB(skb)->end_seq = MPTCP_SKB_CB(skb)->map_seq + copy_len; 306 MPTCP_SKB_CB(skb)->offset = offset; 307 MPTCP_SKB_CB(skb)->has_rxtstamp = has_rxtstamp; 308 309 if (MPTCP_SKB_CB(skb)->map_seq == msk->ack_seq) { 310 /* in sequence */ 311 WRITE_ONCE(msk->ack_seq, msk->ack_seq + copy_len); 312 tail = skb_peek_tail(&sk->sk_receive_queue); 313 if (tail && mptcp_try_coalesce(sk, tail, skb)) 314 return true; 315 316 skb_set_owner_r(skb, sk); 317 __skb_queue_tail(&sk->sk_receive_queue, skb); 318 return true; 319 } else if (after64(MPTCP_SKB_CB(skb)->map_seq, msk->ack_seq)) { 320 mptcp_data_queue_ofo(msk, skb); 321 return false; 322 } 323 324 /* old data, keep it simple and drop the whole pkt, sender 325 * will retransmit as needed, if needed. 326 */ 327 MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_DUPDATA); 328 drop: 329 mptcp_drop(sk, skb); 330 return false; 331 } 332 333 static void mptcp_stop_timer(struct sock *sk) 334 { 335 struct inet_connection_sock *icsk = inet_csk(sk); 336 337 sk_stop_timer(sk, &icsk->icsk_retransmit_timer); 338 mptcp_sk(sk)->timer_ival = 0; 339 } 340 341 static void mptcp_close_wake_up(struct sock *sk) 342 { 343 if (sock_flag(sk, SOCK_DEAD)) 344 return; 345 346 sk->sk_state_change(sk); 347 if (sk->sk_shutdown == SHUTDOWN_MASK || 348 sk->sk_state == TCP_CLOSE) 349 sk_wake_async(sk, SOCK_WAKE_WAITD, POLL_HUP); 350 else 351 sk_wake_async(sk, SOCK_WAKE_WAITD, POLL_IN); 352 } 353 354 static bool mptcp_pending_data_fin_ack(struct sock *sk) 355 { 356 struct mptcp_sock *msk = mptcp_sk(sk); 357 358 return !__mptcp_check_fallback(msk) && 359 ((1 << sk->sk_state) & 360 (TCPF_FIN_WAIT1 | TCPF_CLOSING | TCPF_LAST_ACK)) && 361 msk->write_seq == READ_ONCE(msk->snd_una); 362 } 363 364 static void mptcp_check_data_fin_ack(struct sock *sk) 365 { 366 struct mptcp_sock *msk = mptcp_sk(sk); 367 368 /* Look for an acknowledged DATA_FIN */ 369 if (mptcp_pending_data_fin_ack(sk)) { 370 WRITE_ONCE(msk->snd_data_fin_enable, 0); 371 372 switch (sk->sk_state) { 373 case TCP_FIN_WAIT1: 374 inet_sk_state_store(sk, TCP_FIN_WAIT2); 375 break; 376 case TCP_CLOSING: 377 case TCP_LAST_ACK: 378 inet_sk_state_store(sk, TCP_CLOSE); 379 break; 380 } 381 382 mptcp_close_wake_up(sk); 383 } 384 } 385 386 static bool mptcp_pending_data_fin(struct sock *sk, u64 *seq) 387 { 388 struct mptcp_sock *msk = mptcp_sk(sk); 389 390 if (READ_ONCE(msk->rcv_data_fin) && 391 ((1 << sk->sk_state) & 392 (TCPF_ESTABLISHED | TCPF_FIN_WAIT1 | TCPF_FIN_WAIT2))) { 393 u64 rcv_data_fin_seq = READ_ONCE(msk->rcv_data_fin_seq); 394 395 if (msk->ack_seq == rcv_data_fin_seq) { 396 if (seq) 397 *seq = rcv_data_fin_seq; 398 399 return true; 400 } 401 } 402 403 return false; 404 } 405 406 static void mptcp_set_datafin_timeout(const struct sock *sk) 407 { 408 struct inet_connection_sock *icsk = inet_csk(sk); 409 410 mptcp_sk(sk)->timer_ival = min(TCP_RTO_MAX, 411 TCP_RTO_MIN << icsk->icsk_retransmits); 412 } 413 414 static void mptcp_set_timeout(const struct sock *sk, const struct sock *ssk) 415 { 416 long tout = ssk && inet_csk(ssk)->icsk_pending ? 417 inet_csk(ssk)->icsk_timeout - jiffies : 0; 418 419 if (tout <= 0) 420 tout = mptcp_sk(sk)->timer_ival; 421 mptcp_sk(sk)->timer_ival = tout > 0 ? tout : TCP_RTO_MIN; 422 } 423 424 static bool tcp_can_send_ack(const struct sock *ssk) 425 { 426 return !((1 << inet_sk_state_load(ssk)) & 427 (TCPF_SYN_SENT | TCPF_SYN_RECV | TCPF_TIME_WAIT | TCPF_CLOSE | TCPF_LISTEN)); 428 } 429 430 static void mptcp_send_ack(struct mptcp_sock *msk) 431 { 432 struct mptcp_subflow_context *subflow; 433 434 mptcp_for_each_subflow(msk, subflow) { 435 struct sock *ssk = mptcp_subflow_tcp_sock(subflow); 436 bool slow; 437 438 slow = lock_sock_fast(ssk); 439 if (tcp_can_send_ack(ssk)) 440 tcp_send_ack(ssk); 441 unlock_sock_fast(ssk, slow); 442 } 443 } 444 445 static void mptcp_subflow_cleanup_rbuf(struct sock *ssk) 446 { 447 bool slow; 448 449 slow = lock_sock_fast(ssk); 450 if (tcp_can_send_ack(ssk)) 451 tcp_cleanup_rbuf(ssk, 1); 452 unlock_sock_fast(ssk, slow); 453 } 454 455 static bool mptcp_subflow_could_cleanup(const struct sock *ssk, bool rx_empty) 456 { 457 const struct inet_connection_sock *icsk = inet_csk(ssk); 458 u8 ack_pending = READ_ONCE(icsk->icsk_ack.pending); 459 const struct tcp_sock *tp = tcp_sk(ssk); 460 461 return (ack_pending & ICSK_ACK_SCHED) && 462 ((READ_ONCE(tp->rcv_nxt) - READ_ONCE(tp->rcv_wup) > 463 READ_ONCE(icsk->icsk_ack.rcv_mss)) || 464 (rx_empty && ack_pending & 465 (ICSK_ACK_PUSHED2 | ICSK_ACK_PUSHED))); 466 } 467 468 static void mptcp_cleanup_rbuf(struct mptcp_sock *msk) 469 { 470 int old_space = READ_ONCE(msk->old_wspace); 471 struct mptcp_subflow_context *subflow; 472 struct sock *sk = (struct sock *)msk; 473 int space = __mptcp_space(sk); 474 bool cleanup, rx_empty; 475 476 cleanup = (space > 0) && (space >= (old_space << 1)); 477 rx_empty = !__mptcp_rmem(sk); 478 479 mptcp_for_each_subflow(msk, subflow) { 480 struct sock *ssk = mptcp_subflow_tcp_sock(subflow); 481 482 if (cleanup || mptcp_subflow_could_cleanup(ssk, rx_empty)) 483 mptcp_subflow_cleanup_rbuf(ssk); 484 } 485 } 486 487 static bool mptcp_check_data_fin(struct sock *sk) 488 { 489 struct mptcp_sock *msk = mptcp_sk(sk); 490 u64 rcv_data_fin_seq; 491 bool ret = false; 492 493 if (__mptcp_check_fallback(msk)) 494 return ret; 495 496 /* Need to ack a DATA_FIN received from a peer while this side 497 * of the connection is in ESTABLISHED, FIN_WAIT1, or FIN_WAIT2. 498 * msk->rcv_data_fin was set when parsing the incoming options 499 * at the subflow level and the msk lock was not held, so this 500 * is the first opportunity to act on the DATA_FIN and change 501 * the msk state. 502 * 503 * If we are caught up to the sequence number of the incoming 504 * DATA_FIN, send the DATA_ACK now and do state transition. If 505 * not caught up, do nothing and let the recv code send DATA_ACK 506 * when catching up. 507 */ 508 509 if (mptcp_pending_data_fin(sk, &rcv_data_fin_seq)) { 510 WRITE_ONCE(msk->ack_seq, msk->ack_seq + 1); 511 WRITE_ONCE(msk->rcv_data_fin, 0); 512 513 sk->sk_shutdown |= RCV_SHUTDOWN; 514 smp_mb__before_atomic(); /* SHUTDOWN must be visible first */ 515 set_bit(MPTCP_DATA_READY, &msk->flags); 516 517 switch (sk->sk_state) { 518 case TCP_ESTABLISHED: 519 inet_sk_state_store(sk, TCP_CLOSE_WAIT); 520 break; 521 case TCP_FIN_WAIT1: 522 inet_sk_state_store(sk, TCP_CLOSING); 523 break; 524 case TCP_FIN_WAIT2: 525 inet_sk_state_store(sk, TCP_CLOSE); 526 break; 527 default: 528 /* Other states not expected */ 529 WARN_ON_ONCE(1); 530 break; 531 } 532 533 ret = true; 534 mptcp_set_timeout(sk, NULL); 535 mptcp_send_ack(msk); 536 mptcp_close_wake_up(sk); 537 } 538 return ret; 539 } 540 541 static bool __mptcp_move_skbs_from_subflow(struct mptcp_sock *msk, 542 struct sock *ssk, 543 unsigned int *bytes) 544 { 545 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk); 546 struct sock *sk = (struct sock *)msk; 547 unsigned int moved = 0; 548 bool more_data_avail; 549 struct tcp_sock *tp; 550 bool done = false; 551 int sk_rbuf; 552 553 sk_rbuf = READ_ONCE(sk->sk_rcvbuf); 554 555 if (!(sk->sk_userlocks & SOCK_RCVBUF_LOCK)) { 556 int ssk_rbuf = READ_ONCE(ssk->sk_rcvbuf); 557 558 if (unlikely(ssk_rbuf > sk_rbuf)) { 559 WRITE_ONCE(sk->sk_rcvbuf, ssk_rbuf); 560 sk_rbuf = ssk_rbuf; 561 } 562 } 563 564 pr_debug("msk=%p ssk=%p", msk, ssk); 565 tp = tcp_sk(ssk); 566 do { 567 u32 map_remaining, offset; 568 u32 seq = tp->copied_seq; 569 struct sk_buff *skb; 570 bool fin; 571 572 /* try to move as much data as available */ 573 map_remaining = subflow->map_data_len - 574 mptcp_subflow_get_map_offset(subflow); 575 576 skb = skb_peek(&ssk->sk_receive_queue); 577 if (!skb) { 578 /* if no data is found, a racing workqueue/recvmsg 579 * already processed the new data, stop here or we 580 * can enter an infinite loop 581 */ 582 if (!moved) 583 done = true; 584 break; 585 } 586 587 if (__mptcp_check_fallback(msk)) { 588 /* if we are running under the workqueue, TCP could have 589 * collapsed skbs between dummy map creation and now 590 * be sure to adjust the size 591 */ 592 map_remaining = skb->len; 593 subflow->map_data_len = skb->len; 594 } 595 596 offset = seq - TCP_SKB_CB(skb)->seq; 597 fin = TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN; 598 if (fin) { 599 done = true; 600 seq++; 601 } 602 603 if (offset < skb->len) { 604 size_t len = skb->len - offset; 605 606 if (tp->urg_data) 607 done = true; 608 609 if (__mptcp_move_skb(msk, ssk, skb, offset, len)) 610 moved += len; 611 seq += len; 612 613 if (WARN_ON_ONCE(map_remaining < len)) 614 break; 615 } else { 616 WARN_ON_ONCE(!fin); 617 sk_eat_skb(ssk, skb); 618 done = true; 619 } 620 621 WRITE_ONCE(tp->copied_seq, seq); 622 more_data_avail = mptcp_subflow_data_available(ssk); 623 624 if (atomic_read(&sk->sk_rmem_alloc) > sk_rbuf) { 625 done = true; 626 break; 627 } 628 } while (more_data_avail); 629 630 *bytes += moved; 631 return done; 632 } 633 634 static bool __mptcp_ofo_queue(struct mptcp_sock *msk) 635 { 636 struct sock *sk = (struct sock *)msk; 637 struct sk_buff *skb, *tail; 638 bool moved = false; 639 struct rb_node *p; 640 u64 end_seq; 641 642 p = rb_first(&msk->out_of_order_queue); 643 pr_debug("msk=%p empty=%d", msk, RB_EMPTY_ROOT(&msk->out_of_order_queue)); 644 while (p) { 645 skb = rb_to_skb(p); 646 if (after64(MPTCP_SKB_CB(skb)->map_seq, msk->ack_seq)) 647 break; 648 649 p = rb_next(p); 650 rb_erase(&skb->rbnode, &msk->out_of_order_queue); 651 652 if (unlikely(!after64(MPTCP_SKB_CB(skb)->end_seq, 653 msk->ack_seq))) { 654 mptcp_drop(sk, skb); 655 MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_DUPDATA); 656 continue; 657 } 658 659 end_seq = MPTCP_SKB_CB(skb)->end_seq; 660 tail = skb_peek_tail(&sk->sk_receive_queue); 661 if (!tail || !mptcp_ooo_try_coalesce(msk, tail, skb)) { 662 int delta = msk->ack_seq - MPTCP_SKB_CB(skb)->map_seq; 663 664 /* skip overlapping data, if any */ 665 pr_debug("uncoalesced seq=%llx ack seq=%llx delta=%d", 666 MPTCP_SKB_CB(skb)->map_seq, msk->ack_seq, 667 delta); 668 MPTCP_SKB_CB(skb)->offset += delta; 669 __skb_queue_tail(&sk->sk_receive_queue, skb); 670 } 671 msk->ack_seq = end_seq; 672 moved = true; 673 } 674 return moved; 675 } 676 677 /* In most cases we will be able to lock the mptcp socket. If its already 678 * owned, we need to defer to the work queue to avoid ABBA deadlock. 679 */ 680 static bool move_skbs_to_msk(struct mptcp_sock *msk, struct sock *ssk) 681 { 682 struct sock *sk = (struct sock *)msk; 683 unsigned int moved = 0; 684 685 __mptcp_move_skbs_from_subflow(msk, ssk, &moved); 686 __mptcp_ofo_queue(msk); 687 if (unlikely(ssk->sk_err)) { 688 if (!sock_owned_by_user(sk)) 689 __mptcp_error_report(sk); 690 else 691 set_bit(MPTCP_ERROR_REPORT, &msk->flags); 692 } 693 694 /* If the moves have caught up with the DATA_FIN sequence number 695 * it's time to ack the DATA_FIN and change socket state, but 696 * this is not a good place to change state. Let the workqueue 697 * do it. 698 */ 699 if (mptcp_pending_data_fin(sk, NULL)) 700 mptcp_schedule_work(sk); 701 return moved > 0; 702 } 703 704 void mptcp_data_ready(struct sock *sk, struct sock *ssk) 705 { 706 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk); 707 struct mptcp_sock *msk = mptcp_sk(sk); 708 int sk_rbuf, ssk_rbuf; 709 710 /* The peer can send data while we are shutting down this 711 * subflow at msk destruction time, but we must avoid enqueuing 712 * more data to the msk receive queue 713 */ 714 if (unlikely(subflow->disposable)) 715 return; 716 717 ssk_rbuf = READ_ONCE(ssk->sk_rcvbuf); 718 sk_rbuf = READ_ONCE(sk->sk_rcvbuf); 719 if (unlikely(ssk_rbuf > sk_rbuf)) 720 sk_rbuf = ssk_rbuf; 721 722 /* over limit? can't append more skbs to msk, Also, no need to wake-up*/ 723 if (__mptcp_rmem(sk) > sk_rbuf) { 724 MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_RCVPRUNED); 725 return; 726 } 727 728 /* Wake-up the reader only for in-sequence data */ 729 mptcp_data_lock(sk); 730 if (move_skbs_to_msk(msk, ssk)) { 731 set_bit(MPTCP_DATA_READY, &msk->flags); 732 sk->sk_data_ready(sk); 733 } 734 mptcp_data_unlock(sk); 735 } 736 737 static bool mptcp_do_flush_join_list(struct mptcp_sock *msk) 738 { 739 struct mptcp_subflow_context *subflow; 740 bool ret = false; 741 742 if (likely(list_empty(&msk->join_list))) 743 return false; 744 745 spin_lock_bh(&msk->join_list_lock); 746 list_for_each_entry(subflow, &msk->join_list, node) { 747 u32 sseq = READ_ONCE(subflow->setsockopt_seq); 748 749 mptcp_propagate_sndbuf((struct sock *)msk, mptcp_subflow_tcp_sock(subflow)); 750 if (READ_ONCE(msk->setsockopt_seq) != sseq) 751 ret = true; 752 } 753 list_splice_tail_init(&msk->join_list, &msk->conn_list); 754 spin_unlock_bh(&msk->join_list_lock); 755 756 return ret; 757 } 758 759 void __mptcp_flush_join_list(struct mptcp_sock *msk) 760 { 761 if (likely(!mptcp_do_flush_join_list(msk))) 762 return; 763 764 if (!test_and_set_bit(MPTCP_WORK_SYNC_SETSOCKOPT, &msk->flags)) 765 mptcp_schedule_work((struct sock *)msk); 766 } 767 768 static void mptcp_flush_join_list(struct mptcp_sock *msk) 769 { 770 bool sync_needed = test_and_clear_bit(MPTCP_WORK_SYNC_SETSOCKOPT, &msk->flags); 771 772 might_sleep(); 773 774 if (!mptcp_do_flush_join_list(msk) && !sync_needed) 775 return; 776 777 mptcp_sockopt_sync_all(msk); 778 } 779 780 static bool mptcp_timer_pending(struct sock *sk) 781 { 782 return timer_pending(&inet_csk(sk)->icsk_retransmit_timer); 783 } 784 785 static void mptcp_reset_timer(struct sock *sk) 786 { 787 struct inet_connection_sock *icsk = inet_csk(sk); 788 unsigned long tout; 789 790 /* prevent rescheduling on close */ 791 if (unlikely(inet_sk_state_load(sk) == TCP_CLOSE)) 792 return; 793 794 /* should never be called with mptcp level timer cleared */ 795 tout = READ_ONCE(mptcp_sk(sk)->timer_ival); 796 if (WARN_ON_ONCE(!tout)) 797 tout = TCP_RTO_MIN; 798 sk_reset_timer(sk, &icsk->icsk_retransmit_timer, jiffies + tout); 799 } 800 801 bool mptcp_schedule_work(struct sock *sk) 802 { 803 if (inet_sk_state_load(sk) != TCP_CLOSE && 804 schedule_work(&mptcp_sk(sk)->work)) { 805 /* each subflow already holds a reference to the sk, and the 806 * workqueue is invoked by a subflow, so sk can't go away here. 807 */ 808 sock_hold(sk); 809 return true; 810 } 811 return false; 812 } 813 814 void mptcp_subflow_eof(struct sock *sk) 815 { 816 if (!test_and_set_bit(MPTCP_WORK_EOF, &mptcp_sk(sk)->flags)) 817 mptcp_schedule_work(sk); 818 } 819 820 static void mptcp_check_for_eof(struct mptcp_sock *msk) 821 { 822 struct mptcp_subflow_context *subflow; 823 struct sock *sk = (struct sock *)msk; 824 int receivers = 0; 825 826 mptcp_for_each_subflow(msk, subflow) 827 receivers += !subflow->rx_eof; 828 if (receivers) 829 return; 830 831 if (!(sk->sk_shutdown & RCV_SHUTDOWN)) { 832 /* hopefully temporary hack: propagate shutdown status 833 * to msk, when all subflows agree on it 834 */ 835 sk->sk_shutdown |= RCV_SHUTDOWN; 836 837 smp_mb__before_atomic(); /* SHUTDOWN must be visible first */ 838 set_bit(MPTCP_DATA_READY, &msk->flags); 839 sk->sk_data_ready(sk); 840 } 841 842 switch (sk->sk_state) { 843 case TCP_ESTABLISHED: 844 inet_sk_state_store(sk, TCP_CLOSE_WAIT); 845 break; 846 case TCP_FIN_WAIT1: 847 inet_sk_state_store(sk, TCP_CLOSING); 848 break; 849 case TCP_FIN_WAIT2: 850 inet_sk_state_store(sk, TCP_CLOSE); 851 break; 852 default: 853 return; 854 } 855 mptcp_close_wake_up(sk); 856 } 857 858 static struct sock *mptcp_subflow_recv_lookup(const struct mptcp_sock *msk) 859 { 860 struct mptcp_subflow_context *subflow; 861 struct sock *sk = (struct sock *)msk; 862 863 sock_owned_by_me(sk); 864 865 mptcp_for_each_subflow(msk, subflow) { 866 if (READ_ONCE(subflow->data_avail)) 867 return mptcp_subflow_tcp_sock(subflow); 868 } 869 870 return NULL; 871 } 872 873 static bool mptcp_skb_can_collapse_to(u64 write_seq, 874 const struct sk_buff *skb, 875 const struct mptcp_ext *mpext) 876 { 877 if (!tcp_skb_can_collapse_to(skb)) 878 return false; 879 880 /* can collapse only if MPTCP level sequence is in order and this 881 * mapping has not been xmitted yet 882 */ 883 return mpext && mpext->data_seq + mpext->data_len == write_seq && 884 !mpext->frozen; 885 } 886 887 /* we can append data to the given data frag if: 888 * - there is space available in the backing page_frag 889 * - the data frag tail matches the current page_frag free offset 890 * - the data frag end sequence number matches the current write seq 891 */ 892 static bool mptcp_frag_can_collapse_to(const struct mptcp_sock *msk, 893 const struct page_frag *pfrag, 894 const struct mptcp_data_frag *df) 895 { 896 return df && pfrag->page == df->page && 897 pfrag->size - pfrag->offset > 0 && 898 pfrag->offset == (df->offset + df->data_len) && 899 df->data_seq + df->data_len == msk->write_seq; 900 } 901 902 static int mptcp_wmem_with_overhead(int size) 903 { 904 return size + ((sizeof(struct mptcp_data_frag) * size) >> PAGE_SHIFT); 905 } 906 907 static void __mptcp_wmem_reserve(struct sock *sk, int size) 908 { 909 int amount = mptcp_wmem_with_overhead(size); 910 struct mptcp_sock *msk = mptcp_sk(sk); 911 912 WARN_ON_ONCE(msk->wmem_reserved); 913 if (WARN_ON_ONCE(amount < 0)) 914 amount = 0; 915 916 if (amount <= sk->sk_forward_alloc) 917 goto reserve; 918 919 /* under memory pressure try to reserve at most a single page 920 * otherwise try to reserve the full estimate and fallback 921 * to a single page before entering the error path 922 */ 923 if ((tcp_under_memory_pressure(sk) && amount > PAGE_SIZE) || 924 !sk_wmem_schedule(sk, amount)) { 925 if (amount <= PAGE_SIZE) 926 goto nomem; 927 928 amount = PAGE_SIZE; 929 if (!sk_wmem_schedule(sk, amount)) 930 goto nomem; 931 } 932 933 reserve: 934 msk->wmem_reserved = amount; 935 sk->sk_forward_alloc -= amount; 936 return; 937 938 nomem: 939 /* we will wait for memory on next allocation */ 940 msk->wmem_reserved = -1; 941 } 942 943 static void __mptcp_update_wmem(struct sock *sk) 944 { 945 struct mptcp_sock *msk = mptcp_sk(sk); 946 947 #ifdef CONFIG_LOCKDEP 948 WARN_ON_ONCE(!lockdep_is_held(&sk->sk_lock.slock)); 949 #endif 950 951 if (!msk->wmem_reserved) 952 return; 953 954 if (msk->wmem_reserved < 0) 955 msk->wmem_reserved = 0; 956 if (msk->wmem_reserved > 0) { 957 sk->sk_forward_alloc += msk->wmem_reserved; 958 msk->wmem_reserved = 0; 959 } 960 } 961 962 static bool mptcp_wmem_alloc(struct sock *sk, int size) 963 { 964 struct mptcp_sock *msk = mptcp_sk(sk); 965 966 /* check for pre-existing error condition */ 967 if (msk->wmem_reserved < 0) 968 return false; 969 970 if (msk->wmem_reserved >= size) 971 goto account; 972 973 mptcp_data_lock(sk); 974 if (!sk_wmem_schedule(sk, size)) { 975 mptcp_data_unlock(sk); 976 return false; 977 } 978 979 sk->sk_forward_alloc -= size; 980 msk->wmem_reserved += size; 981 mptcp_data_unlock(sk); 982 983 account: 984 msk->wmem_reserved -= size; 985 return true; 986 } 987 988 static void mptcp_wmem_uncharge(struct sock *sk, int size) 989 { 990 struct mptcp_sock *msk = mptcp_sk(sk); 991 992 if (msk->wmem_reserved < 0) 993 msk->wmem_reserved = 0; 994 msk->wmem_reserved += size; 995 } 996 997 static void mptcp_mem_reclaim_partial(struct sock *sk) 998 { 999 struct mptcp_sock *msk = mptcp_sk(sk); 1000 1001 /* if we are experiencing a transint allocation error, 1002 * the forward allocation memory has been already 1003 * released 1004 */ 1005 if (msk->wmem_reserved < 0) 1006 return; 1007 1008 mptcp_data_lock(sk); 1009 sk->sk_forward_alloc += msk->wmem_reserved; 1010 sk_mem_reclaim_partial(sk); 1011 msk->wmem_reserved = sk->sk_forward_alloc; 1012 sk->sk_forward_alloc = 0; 1013 mptcp_data_unlock(sk); 1014 } 1015 1016 static void dfrag_uncharge(struct sock *sk, int len) 1017 { 1018 sk_mem_uncharge(sk, len); 1019 sk_wmem_queued_add(sk, -len); 1020 } 1021 1022 static void dfrag_clear(struct sock *sk, struct mptcp_data_frag *dfrag) 1023 { 1024 int len = dfrag->data_len + dfrag->overhead; 1025 1026 list_del(&dfrag->list); 1027 dfrag_uncharge(sk, len); 1028 put_page(dfrag->page); 1029 } 1030 1031 static void __mptcp_clean_una(struct sock *sk) 1032 { 1033 struct mptcp_sock *msk = mptcp_sk(sk); 1034 struct mptcp_data_frag *dtmp, *dfrag; 1035 bool cleaned = false; 1036 u64 snd_una; 1037 1038 /* on fallback we just need to ignore snd_una, as this is really 1039 * plain TCP 1040 */ 1041 if (__mptcp_check_fallback(msk)) 1042 msk->snd_una = READ_ONCE(msk->snd_nxt); 1043 1044 snd_una = msk->snd_una; 1045 list_for_each_entry_safe(dfrag, dtmp, &msk->rtx_queue, list) { 1046 if (after64(dfrag->data_seq + dfrag->data_len, snd_una)) 1047 break; 1048 1049 if (WARN_ON_ONCE(dfrag == msk->first_pending)) 1050 break; 1051 dfrag_clear(sk, dfrag); 1052 cleaned = true; 1053 } 1054 1055 dfrag = mptcp_rtx_head(sk); 1056 if (dfrag && after64(snd_una, dfrag->data_seq)) { 1057 u64 delta = snd_una - dfrag->data_seq; 1058 1059 if (WARN_ON_ONCE(delta > dfrag->already_sent)) 1060 goto out; 1061 1062 dfrag->data_seq += delta; 1063 dfrag->offset += delta; 1064 dfrag->data_len -= delta; 1065 dfrag->already_sent -= delta; 1066 1067 dfrag_uncharge(sk, delta); 1068 cleaned = true; 1069 } 1070 1071 out: 1072 if (cleaned) { 1073 if (tcp_under_memory_pressure(sk)) { 1074 __mptcp_update_wmem(sk); 1075 sk_mem_reclaim_partial(sk); 1076 } 1077 } 1078 1079 if (snd_una == READ_ONCE(msk->snd_nxt)) { 1080 if (msk->timer_ival && !mptcp_data_fin_enabled(msk)) 1081 mptcp_stop_timer(sk); 1082 } else { 1083 mptcp_reset_timer(sk); 1084 } 1085 } 1086 1087 static void __mptcp_clean_una_wakeup(struct sock *sk) 1088 { 1089 #ifdef CONFIG_LOCKDEP 1090 WARN_ON_ONCE(!lockdep_is_held(&sk->sk_lock.slock)); 1091 #endif 1092 __mptcp_clean_una(sk); 1093 mptcp_write_space(sk); 1094 } 1095 1096 static void mptcp_clean_una_wakeup(struct sock *sk) 1097 { 1098 mptcp_data_lock(sk); 1099 __mptcp_clean_una_wakeup(sk); 1100 mptcp_data_unlock(sk); 1101 } 1102 1103 static void mptcp_enter_memory_pressure(struct sock *sk) 1104 { 1105 struct mptcp_subflow_context *subflow; 1106 struct mptcp_sock *msk = mptcp_sk(sk); 1107 bool first = true; 1108 1109 sk_stream_moderate_sndbuf(sk); 1110 mptcp_for_each_subflow(msk, subflow) { 1111 struct sock *ssk = mptcp_subflow_tcp_sock(subflow); 1112 1113 if (first) 1114 tcp_enter_memory_pressure(ssk); 1115 sk_stream_moderate_sndbuf(ssk); 1116 first = false; 1117 } 1118 } 1119 1120 /* ensure we get enough memory for the frag hdr, beyond some minimal amount of 1121 * data 1122 */ 1123 static bool mptcp_page_frag_refill(struct sock *sk, struct page_frag *pfrag) 1124 { 1125 if (likely(skb_page_frag_refill(32U + sizeof(struct mptcp_data_frag), 1126 pfrag, sk->sk_allocation))) 1127 return true; 1128 1129 mptcp_enter_memory_pressure(sk); 1130 return false; 1131 } 1132 1133 static struct mptcp_data_frag * 1134 mptcp_carve_data_frag(const struct mptcp_sock *msk, struct page_frag *pfrag, 1135 int orig_offset) 1136 { 1137 int offset = ALIGN(orig_offset, sizeof(long)); 1138 struct mptcp_data_frag *dfrag; 1139 1140 dfrag = (struct mptcp_data_frag *)(page_to_virt(pfrag->page) + offset); 1141 dfrag->data_len = 0; 1142 dfrag->data_seq = msk->write_seq; 1143 dfrag->overhead = offset - orig_offset + sizeof(struct mptcp_data_frag); 1144 dfrag->offset = offset + sizeof(struct mptcp_data_frag); 1145 dfrag->already_sent = 0; 1146 dfrag->page = pfrag->page; 1147 1148 return dfrag; 1149 } 1150 1151 struct mptcp_sendmsg_info { 1152 int mss_now; 1153 int size_goal; 1154 u16 limit; 1155 u16 sent; 1156 unsigned int flags; 1157 }; 1158 1159 static int mptcp_check_allowed_size(struct mptcp_sock *msk, u64 data_seq, 1160 int avail_size) 1161 { 1162 u64 window_end = mptcp_wnd_end(msk); 1163 1164 if (__mptcp_check_fallback(msk)) 1165 return avail_size; 1166 1167 if (!before64(data_seq + avail_size, window_end)) { 1168 u64 allowed_size = window_end - data_seq; 1169 1170 return min_t(unsigned int, allowed_size, avail_size); 1171 } 1172 1173 return avail_size; 1174 } 1175 1176 static bool __mptcp_add_ext(struct sk_buff *skb, gfp_t gfp) 1177 { 1178 struct skb_ext *mpext = __skb_ext_alloc(gfp); 1179 1180 if (!mpext) 1181 return false; 1182 __skb_ext_set(skb, SKB_EXT_MPTCP, mpext); 1183 return true; 1184 } 1185 1186 static struct sk_buff *__mptcp_do_alloc_tx_skb(struct sock *sk, gfp_t gfp) 1187 { 1188 struct sk_buff *skb; 1189 1190 skb = alloc_skb_fclone(MAX_TCP_HEADER, gfp); 1191 if (likely(skb)) { 1192 if (likely(__mptcp_add_ext(skb, gfp))) { 1193 skb_reserve(skb, MAX_TCP_HEADER); 1194 skb->reserved_tailroom = skb->end - skb->tail; 1195 return skb; 1196 } 1197 __kfree_skb(skb); 1198 } else { 1199 mptcp_enter_memory_pressure(sk); 1200 } 1201 return NULL; 1202 } 1203 1204 static bool __mptcp_alloc_tx_skb(struct sock *sk, struct sock *ssk, gfp_t gfp) 1205 { 1206 struct sk_buff *skb; 1207 1208 if (ssk->sk_tx_skb_cache) { 1209 skb = ssk->sk_tx_skb_cache; 1210 if (unlikely(!skb_ext_find(skb, SKB_EXT_MPTCP) && 1211 !__mptcp_add_ext(skb, gfp))) 1212 return false; 1213 return true; 1214 } 1215 1216 skb = __mptcp_do_alloc_tx_skb(sk, gfp); 1217 if (!skb) 1218 return false; 1219 1220 if (likely(sk_wmem_schedule(ssk, skb->truesize))) { 1221 ssk->sk_tx_skb_cache = skb; 1222 return true; 1223 } 1224 kfree_skb(skb); 1225 return false; 1226 } 1227 1228 static bool mptcp_must_reclaim_memory(struct sock *sk, struct sock *ssk) 1229 { 1230 return !ssk->sk_tx_skb_cache && 1231 tcp_under_memory_pressure(sk); 1232 } 1233 1234 static bool mptcp_alloc_tx_skb(struct sock *sk, struct sock *ssk) 1235 { 1236 if (unlikely(mptcp_must_reclaim_memory(sk, ssk))) 1237 mptcp_mem_reclaim_partial(sk); 1238 return __mptcp_alloc_tx_skb(sk, ssk, sk->sk_allocation); 1239 } 1240 1241 /* note: this always recompute the csum on the whole skb, even 1242 * if we just appended a single frag. More status info needed 1243 */ 1244 static void mptcp_update_data_checksum(struct sk_buff *skb, int added) 1245 { 1246 struct mptcp_ext *mpext = mptcp_get_ext(skb); 1247 __wsum csum = ~csum_unfold(mpext->csum); 1248 int offset = skb->len - added; 1249 1250 mpext->csum = csum_fold(csum_block_add(csum, skb_checksum(skb, offset, added, 0), offset)); 1251 } 1252 1253 static int mptcp_sendmsg_frag(struct sock *sk, struct sock *ssk, 1254 struct mptcp_data_frag *dfrag, 1255 struct mptcp_sendmsg_info *info) 1256 { 1257 u64 data_seq = dfrag->data_seq + info->sent; 1258 struct mptcp_sock *msk = mptcp_sk(sk); 1259 bool zero_window_probe = false; 1260 struct mptcp_ext *mpext = NULL; 1261 struct sk_buff *skb, *tail; 1262 bool can_collapse = false; 1263 int size_bias = 0; 1264 int avail_size; 1265 size_t ret = 0; 1266 1267 pr_debug("msk=%p ssk=%p sending dfrag at seq=%llu len=%u already sent=%u", 1268 msk, ssk, dfrag->data_seq, dfrag->data_len, info->sent); 1269 1270 /* compute send limit */ 1271 info->mss_now = tcp_send_mss(ssk, &info->size_goal, info->flags); 1272 avail_size = info->size_goal; 1273 skb = tcp_write_queue_tail(ssk); 1274 if (skb) { 1275 /* Limit the write to the size available in the 1276 * current skb, if any, so that we create at most a new skb. 1277 * Explicitly tells TCP internals to avoid collapsing on later 1278 * queue management operation, to avoid breaking the ext <-> 1279 * SSN association set here 1280 */ 1281 mpext = skb_ext_find(skb, SKB_EXT_MPTCP); 1282 can_collapse = (info->size_goal - skb->len > 0) && 1283 mptcp_skb_can_collapse_to(data_seq, skb, mpext); 1284 if (!can_collapse) { 1285 TCP_SKB_CB(skb)->eor = 1; 1286 } else { 1287 size_bias = skb->len; 1288 avail_size = info->size_goal - skb->len; 1289 } 1290 } 1291 1292 /* Zero window and all data acked? Probe. */ 1293 avail_size = mptcp_check_allowed_size(msk, data_seq, avail_size); 1294 if (avail_size == 0) { 1295 u64 snd_una = READ_ONCE(msk->snd_una); 1296 1297 if (skb || snd_una != msk->snd_nxt) 1298 return 0; 1299 zero_window_probe = true; 1300 data_seq = snd_una - 1; 1301 avail_size = 1; 1302 } 1303 1304 if (WARN_ON_ONCE(info->sent > info->limit || 1305 info->limit > dfrag->data_len)) 1306 return 0; 1307 1308 ret = info->limit - info->sent; 1309 tail = tcp_build_frag(ssk, avail_size + size_bias, info->flags, 1310 dfrag->page, dfrag->offset + info->sent, &ret); 1311 if (!tail) { 1312 tcp_remove_empty_skb(sk, tcp_write_queue_tail(ssk)); 1313 return -ENOMEM; 1314 } 1315 1316 /* if the tail skb is still the cached one, collapsing really happened. 1317 */ 1318 if (skb == tail) { 1319 TCP_SKB_CB(tail)->tcp_flags &= ~TCPHDR_PSH; 1320 mpext->data_len += ret; 1321 WARN_ON_ONCE(!can_collapse); 1322 WARN_ON_ONCE(zero_window_probe); 1323 goto out; 1324 } 1325 1326 mpext = skb_ext_find(tail, SKB_EXT_MPTCP); 1327 if (WARN_ON_ONCE(!mpext)) { 1328 /* should never reach here, stream corrupted */ 1329 return -EINVAL; 1330 } 1331 1332 memset(mpext, 0, sizeof(*mpext)); 1333 mpext->data_seq = data_seq; 1334 mpext->subflow_seq = mptcp_subflow_ctx(ssk)->rel_write_seq; 1335 mpext->data_len = ret; 1336 mpext->use_map = 1; 1337 mpext->dsn64 = 1; 1338 1339 pr_debug("data_seq=%llu subflow_seq=%u data_len=%u dsn64=%d", 1340 mpext->data_seq, mpext->subflow_seq, mpext->data_len, 1341 mpext->dsn64); 1342 1343 if (zero_window_probe) { 1344 mptcp_subflow_ctx(ssk)->rel_write_seq += ret; 1345 mpext->frozen = 1; 1346 if (READ_ONCE(msk->csum_enabled)) 1347 mptcp_update_data_checksum(tail, ret); 1348 tcp_push_pending_frames(ssk); 1349 return 0; 1350 } 1351 out: 1352 if (READ_ONCE(msk->csum_enabled)) 1353 mptcp_update_data_checksum(tail, ret); 1354 mptcp_subflow_ctx(ssk)->rel_write_seq += ret; 1355 return ret; 1356 } 1357 1358 #define MPTCP_SEND_BURST_SIZE ((1 << 16) - \ 1359 sizeof(struct tcphdr) - \ 1360 MAX_TCP_OPTION_SPACE - \ 1361 sizeof(struct ipv6hdr) - \ 1362 sizeof(struct frag_hdr)) 1363 1364 struct subflow_send_info { 1365 struct sock *ssk; 1366 u64 ratio; 1367 }; 1368 1369 static struct sock *mptcp_subflow_get_send(struct mptcp_sock *msk) 1370 { 1371 struct subflow_send_info send_info[2]; 1372 struct mptcp_subflow_context *subflow; 1373 int i, nr_active = 0; 1374 struct sock *ssk; 1375 u64 ratio; 1376 u32 pace; 1377 1378 sock_owned_by_me((struct sock *)msk); 1379 1380 if (__mptcp_check_fallback(msk)) { 1381 if (!msk->first) 1382 return NULL; 1383 return sk_stream_memory_free(msk->first) ? msk->first : NULL; 1384 } 1385 1386 /* re-use last subflow, if the burst allow that */ 1387 if (msk->last_snd && msk->snd_burst > 0 && 1388 sk_stream_memory_free(msk->last_snd) && 1389 mptcp_subflow_active(mptcp_subflow_ctx(msk->last_snd))) 1390 return msk->last_snd; 1391 1392 /* pick the subflow with the lower wmem/wspace ratio */ 1393 for (i = 0; i < 2; ++i) { 1394 send_info[i].ssk = NULL; 1395 send_info[i].ratio = -1; 1396 } 1397 mptcp_for_each_subflow(msk, subflow) { 1398 trace_mptcp_subflow_get_send(subflow); 1399 ssk = mptcp_subflow_tcp_sock(subflow); 1400 if (!mptcp_subflow_active(subflow)) 1401 continue; 1402 1403 nr_active += !subflow->backup; 1404 if (!sk_stream_memory_free(subflow->tcp_sock) || !tcp_sk(ssk)->snd_wnd) 1405 continue; 1406 1407 pace = READ_ONCE(ssk->sk_pacing_rate); 1408 if (!pace) 1409 continue; 1410 1411 ratio = div_u64((u64)READ_ONCE(ssk->sk_wmem_queued) << 32, 1412 pace); 1413 if (ratio < send_info[subflow->backup].ratio) { 1414 send_info[subflow->backup].ssk = ssk; 1415 send_info[subflow->backup].ratio = ratio; 1416 } 1417 } 1418 1419 /* pick the best backup if no other subflow is active */ 1420 if (!nr_active) 1421 send_info[0].ssk = send_info[1].ssk; 1422 1423 if (send_info[0].ssk) { 1424 msk->last_snd = send_info[0].ssk; 1425 msk->snd_burst = min_t(int, MPTCP_SEND_BURST_SIZE, 1426 tcp_sk(msk->last_snd)->snd_wnd); 1427 return msk->last_snd; 1428 } 1429 1430 return NULL; 1431 } 1432 1433 static void mptcp_push_release(struct sock *sk, struct sock *ssk, 1434 struct mptcp_sendmsg_info *info) 1435 { 1436 mptcp_set_timeout(sk, ssk); 1437 tcp_push(ssk, 0, info->mss_now, tcp_sk(ssk)->nonagle, info->size_goal); 1438 release_sock(ssk); 1439 } 1440 1441 static void __mptcp_push_pending(struct sock *sk, unsigned int flags) 1442 { 1443 struct sock *prev_ssk = NULL, *ssk = NULL; 1444 struct mptcp_sock *msk = mptcp_sk(sk); 1445 struct mptcp_sendmsg_info info = { 1446 .flags = flags, 1447 }; 1448 struct mptcp_data_frag *dfrag; 1449 int len, copied = 0; 1450 1451 while ((dfrag = mptcp_send_head(sk))) { 1452 info.sent = dfrag->already_sent; 1453 info.limit = dfrag->data_len; 1454 len = dfrag->data_len - dfrag->already_sent; 1455 while (len > 0) { 1456 int ret = 0; 1457 1458 prev_ssk = ssk; 1459 mptcp_flush_join_list(msk); 1460 ssk = mptcp_subflow_get_send(msk); 1461 1462 /* try to keep the subflow socket lock across 1463 * consecutive xmit on the same socket 1464 */ 1465 if (ssk != prev_ssk && prev_ssk) 1466 mptcp_push_release(sk, prev_ssk, &info); 1467 if (!ssk) 1468 goto out; 1469 1470 if (ssk != prev_ssk || !prev_ssk) 1471 lock_sock(ssk); 1472 1473 /* keep it simple and always provide a new skb for the 1474 * subflow, even if we will not use it when collapsing 1475 * on the pending one 1476 */ 1477 if (!mptcp_alloc_tx_skb(sk, ssk)) { 1478 mptcp_push_release(sk, ssk, &info); 1479 goto out; 1480 } 1481 1482 ret = mptcp_sendmsg_frag(sk, ssk, dfrag, &info); 1483 if (ret <= 0) { 1484 mptcp_push_release(sk, ssk, &info); 1485 goto out; 1486 } 1487 1488 info.sent += ret; 1489 dfrag->already_sent += ret; 1490 msk->snd_nxt += ret; 1491 msk->snd_burst -= ret; 1492 msk->tx_pending_data -= ret; 1493 copied += ret; 1494 len -= ret; 1495 } 1496 WRITE_ONCE(msk->first_pending, mptcp_send_next(sk)); 1497 } 1498 1499 /* at this point we held the socket lock for the last subflow we used */ 1500 if (ssk) 1501 mptcp_push_release(sk, ssk, &info); 1502 1503 out: 1504 if (copied) { 1505 /* start the timer, if it's not pending */ 1506 if (!mptcp_timer_pending(sk)) 1507 mptcp_reset_timer(sk); 1508 __mptcp_check_send_data_fin(sk); 1509 } 1510 } 1511 1512 static void __mptcp_subflow_push_pending(struct sock *sk, struct sock *ssk) 1513 { 1514 struct mptcp_sock *msk = mptcp_sk(sk); 1515 struct mptcp_sendmsg_info info; 1516 struct mptcp_data_frag *dfrag; 1517 struct sock *xmit_ssk; 1518 int len, copied = 0; 1519 bool first = true; 1520 1521 info.flags = 0; 1522 while ((dfrag = mptcp_send_head(sk))) { 1523 info.sent = dfrag->already_sent; 1524 info.limit = dfrag->data_len; 1525 len = dfrag->data_len - dfrag->already_sent; 1526 while (len > 0) { 1527 int ret = 0; 1528 1529 /* the caller already invoked the packet scheduler, 1530 * check for a different subflow usage only after 1531 * spooling the first chunk of data 1532 */ 1533 xmit_ssk = first ? ssk : mptcp_subflow_get_send(mptcp_sk(sk)); 1534 if (!xmit_ssk) 1535 goto out; 1536 if (xmit_ssk != ssk) { 1537 mptcp_subflow_delegate(mptcp_subflow_ctx(xmit_ssk)); 1538 goto out; 1539 } 1540 1541 if (unlikely(mptcp_must_reclaim_memory(sk, ssk))) { 1542 __mptcp_update_wmem(sk); 1543 sk_mem_reclaim_partial(sk); 1544 } 1545 if (!__mptcp_alloc_tx_skb(sk, ssk, GFP_ATOMIC)) 1546 goto out; 1547 1548 ret = mptcp_sendmsg_frag(sk, ssk, dfrag, &info); 1549 if (ret <= 0) 1550 goto out; 1551 1552 info.sent += ret; 1553 dfrag->already_sent += ret; 1554 msk->snd_nxt += ret; 1555 msk->snd_burst -= ret; 1556 msk->tx_pending_data -= ret; 1557 copied += ret; 1558 len -= ret; 1559 first = false; 1560 } 1561 WRITE_ONCE(msk->first_pending, mptcp_send_next(sk)); 1562 } 1563 1564 out: 1565 /* __mptcp_alloc_tx_skb could have released some wmem and we are 1566 * not going to flush it via release_sock() 1567 */ 1568 __mptcp_update_wmem(sk); 1569 if (copied) { 1570 mptcp_set_timeout(sk, ssk); 1571 tcp_push(ssk, 0, info.mss_now, tcp_sk(ssk)->nonagle, 1572 info.size_goal); 1573 if (!mptcp_timer_pending(sk)) 1574 mptcp_reset_timer(sk); 1575 1576 if (msk->snd_data_fin_enable && 1577 msk->snd_nxt + 1 == msk->write_seq) 1578 mptcp_schedule_work(sk); 1579 } 1580 } 1581 1582 static void mptcp_set_nospace(struct sock *sk) 1583 { 1584 /* enable autotune */ 1585 set_bit(SOCK_NOSPACE, &sk->sk_socket->flags); 1586 1587 /* will be cleared on avail space */ 1588 set_bit(MPTCP_NOSPACE, &mptcp_sk(sk)->flags); 1589 } 1590 1591 static int mptcp_sendmsg(struct sock *sk, struct msghdr *msg, size_t len) 1592 { 1593 struct mptcp_sock *msk = mptcp_sk(sk); 1594 struct page_frag *pfrag; 1595 size_t copied = 0; 1596 int ret = 0; 1597 long timeo; 1598 1599 /* we don't support FASTOPEN yet */ 1600 if (msg->msg_flags & MSG_FASTOPEN) 1601 return -EOPNOTSUPP; 1602 1603 /* silently ignore everything else */ 1604 msg->msg_flags &= MSG_MORE | MSG_DONTWAIT | MSG_NOSIGNAL; 1605 1606 mptcp_lock_sock(sk, __mptcp_wmem_reserve(sk, min_t(size_t, 1 << 20, len))); 1607 1608 timeo = sock_sndtimeo(sk, msg->msg_flags & MSG_DONTWAIT); 1609 1610 if ((1 << sk->sk_state) & ~(TCPF_ESTABLISHED | TCPF_CLOSE_WAIT)) { 1611 ret = sk_stream_wait_connect(sk, &timeo); 1612 if (ret) 1613 goto out; 1614 } 1615 1616 pfrag = sk_page_frag(sk); 1617 1618 while (msg_data_left(msg)) { 1619 int total_ts, frag_truesize = 0; 1620 struct mptcp_data_frag *dfrag; 1621 bool dfrag_collapsed; 1622 size_t psize, offset; 1623 1624 if (sk->sk_err || (sk->sk_shutdown & SEND_SHUTDOWN)) { 1625 ret = -EPIPE; 1626 goto out; 1627 } 1628 1629 /* reuse tail pfrag, if possible, or carve a new one from the 1630 * page allocator 1631 */ 1632 dfrag = mptcp_pending_tail(sk); 1633 dfrag_collapsed = mptcp_frag_can_collapse_to(msk, pfrag, dfrag); 1634 if (!dfrag_collapsed) { 1635 if (!sk_stream_memory_free(sk)) 1636 goto wait_for_memory; 1637 1638 if (!mptcp_page_frag_refill(sk, pfrag)) 1639 goto wait_for_memory; 1640 1641 dfrag = mptcp_carve_data_frag(msk, pfrag, pfrag->offset); 1642 frag_truesize = dfrag->overhead; 1643 } 1644 1645 /* we do not bound vs wspace, to allow a single packet. 1646 * memory accounting will prevent execessive memory usage 1647 * anyway 1648 */ 1649 offset = dfrag->offset + dfrag->data_len; 1650 psize = pfrag->size - offset; 1651 psize = min_t(size_t, psize, msg_data_left(msg)); 1652 total_ts = psize + frag_truesize; 1653 1654 if (!mptcp_wmem_alloc(sk, total_ts)) 1655 goto wait_for_memory; 1656 1657 if (copy_page_from_iter(dfrag->page, offset, psize, 1658 &msg->msg_iter) != psize) { 1659 mptcp_wmem_uncharge(sk, psize + frag_truesize); 1660 ret = -EFAULT; 1661 goto out; 1662 } 1663 1664 /* data successfully copied into the write queue */ 1665 copied += psize; 1666 dfrag->data_len += psize; 1667 frag_truesize += psize; 1668 pfrag->offset += frag_truesize; 1669 WRITE_ONCE(msk->write_seq, msk->write_seq + psize); 1670 msk->tx_pending_data += psize; 1671 1672 /* charge data on mptcp pending queue to the msk socket 1673 * Note: we charge such data both to sk and ssk 1674 */ 1675 sk_wmem_queued_add(sk, frag_truesize); 1676 if (!dfrag_collapsed) { 1677 get_page(dfrag->page); 1678 list_add_tail(&dfrag->list, &msk->rtx_queue); 1679 if (!msk->first_pending) 1680 WRITE_ONCE(msk->first_pending, dfrag); 1681 } 1682 pr_debug("msk=%p dfrag at seq=%llu len=%u sent=%u new=%d", msk, 1683 dfrag->data_seq, dfrag->data_len, dfrag->already_sent, 1684 !dfrag_collapsed); 1685 1686 continue; 1687 1688 wait_for_memory: 1689 mptcp_set_nospace(sk); 1690 __mptcp_push_pending(sk, msg->msg_flags); 1691 ret = sk_stream_wait_memory(sk, &timeo); 1692 if (ret) 1693 goto out; 1694 } 1695 1696 if (copied) 1697 __mptcp_push_pending(sk, msg->msg_flags); 1698 1699 out: 1700 release_sock(sk); 1701 return copied ? : ret; 1702 } 1703 1704 static void mptcp_wait_data(struct sock *sk, long *timeo) 1705 { 1706 DEFINE_WAIT_FUNC(wait, woken_wake_function); 1707 struct mptcp_sock *msk = mptcp_sk(sk); 1708 1709 add_wait_queue(sk_sleep(sk), &wait); 1710 sk_set_bit(SOCKWQ_ASYNC_WAITDATA, sk); 1711 1712 sk_wait_event(sk, timeo, 1713 test_bit(MPTCP_DATA_READY, &msk->flags), &wait); 1714 1715 sk_clear_bit(SOCKWQ_ASYNC_WAITDATA, sk); 1716 remove_wait_queue(sk_sleep(sk), &wait); 1717 } 1718 1719 static int __mptcp_recvmsg_mskq(struct mptcp_sock *msk, 1720 struct msghdr *msg, 1721 size_t len, int flags, 1722 struct scm_timestamping_internal *tss, 1723 int *cmsg_flags) 1724 { 1725 struct sk_buff *skb, *tmp; 1726 int copied = 0; 1727 1728 skb_queue_walk_safe(&msk->receive_queue, skb, tmp) { 1729 u32 offset = MPTCP_SKB_CB(skb)->offset; 1730 u32 data_len = skb->len - offset; 1731 u32 count = min_t(size_t, len - copied, data_len); 1732 int err; 1733 1734 if (!(flags & MSG_TRUNC)) { 1735 err = skb_copy_datagram_msg(skb, offset, msg, count); 1736 if (unlikely(err < 0)) { 1737 if (!copied) 1738 return err; 1739 break; 1740 } 1741 } 1742 1743 if (MPTCP_SKB_CB(skb)->has_rxtstamp) { 1744 tcp_update_recv_tstamps(skb, tss); 1745 *cmsg_flags |= MPTCP_CMSG_TS; 1746 } 1747 1748 copied += count; 1749 1750 if (count < data_len) { 1751 if (!(flags & MSG_PEEK)) 1752 MPTCP_SKB_CB(skb)->offset += count; 1753 break; 1754 } 1755 1756 if (!(flags & MSG_PEEK)) { 1757 /* we will bulk release the skb memory later */ 1758 skb->destructor = NULL; 1759 WRITE_ONCE(msk->rmem_released, msk->rmem_released + skb->truesize); 1760 __skb_unlink(skb, &msk->receive_queue); 1761 __kfree_skb(skb); 1762 } 1763 1764 if (copied >= len) 1765 break; 1766 } 1767 1768 return copied; 1769 } 1770 1771 /* receive buffer autotuning. See tcp_rcv_space_adjust for more information. 1772 * 1773 * Only difference: Use highest rtt estimate of the subflows in use. 1774 */ 1775 static void mptcp_rcv_space_adjust(struct mptcp_sock *msk, int copied) 1776 { 1777 struct mptcp_subflow_context *subflow; 1778 struct sock *sk = (struct sock *)msk; 1779 u32 time, advmss = 1; 1780 u64 rtt_us, mstamp; 1781 1782 sock_owned_by_me(sk); 1783 1784 if (copied <= 0) 1785 return; 1786 1787 msk->rcvq_space.copied += copied; 1788 1789 mstamp = div_u64(tcp_clock_ns(), NSEC_PER_USEC); 1790 time = tcp_stamp_us_delta(mstamp, msk->rcvq_space.time); 1791 1792 rtt_us = msk->rcvq_space.rtt_us; 1793 if (rtt_us && time < (rtt_us >> 3)) 1794 return; 1795 1796 rtt_us = 0; 1797 mptcp_for_each_subflow(msk, subflow) { 1798 const struct tcp_sock *tp; 1799 u64 sf_rtt_us; 1800 u32 sf_advmss; 1801 1802 tp = tcp_sk(mptcp_subflow_tcp_sock(subflow)); 1803 1804 sf_rtt_us = READ_ONCE(tp->rcv_rtt_est.rtt_us); 1805 sf_advmss = READ_ONCE(tp->advmss); 1806 1807 rtt_us = max(sf_rtt_us, rtt_us); 1808 advmss = max(sf_advmss, advmss); 1809 } 1810 1811 msk->rcvq_space.rtt_us = rtt_us; 1812 if (time < (rtt_us >> 3) || rtt_us == 0) 1813 return; 1814 1815 if (msk->rcvq_space.copied <= msk->rcvq_space.space) 1816 goto new_measure; 1817 1818 if (sock_net(sk)->ipv4.sysctl_tcp_moderate_rcvbuf && 1819 !(sk->sk_userlocks & SOCK_RCVBUF_LOCK)) { 1820 int rcvmem, rcvbuf; 1821 u64 rcvwin, grow; 1822 1823 rcvwin = ((u64)msk->rcvq_space.copied << 1) + 16 * advmss; 1824 1825 grow = rcvwin * (msk->rcvq_space.copied - msk->rcvq_space.space); 1826 1827 do_div(grow, msk->rcvq_space.space); 1828 rcvwin += (grow << 1); 1829 1830 rcvmem = SKB_TRUESIZE(advmss + MAX_TCP_HEADER); 1831 while (tcp_win_from_space(sk, rcvmem) < advmss) 1832 rcvmem += 128; 1833 1834 do_div(rcvwin, advmss); 1835 rcvbuf = min_t(u64, rcvwin * rcvmem, 1836 sock_net(sk)->ipv4.sysctl_tcp_rmem[2]); 1837 1838 if (rcvbuf > sk->sk_rcvbuf) { 1839 u32 window_clamp; 1840 1841 window_clamp = tcp_win_from_space(sk, rcvbuf); 1842 WRITE_ONCE(sk->sk_rcvbuf, rcvbuf); 1843 1844 /* Make subflows follow along. If we do not do this, we 1845 * get drops at subflow level if skbs can't be moved to 1846 * the mptcp rx queue fast enough (announced rcv_win can 1847 * exceed ssk->sk_rcvbuf). 1848 */ 1849 mptcp_for_each_subflow(msk, subflow) { 1850 struct sock *ssk; 1851 bool slow; 1852 1853 ssk = mptcp_subflow_tcp_sock(subflow); 1854 slow = lock_sock_fast(ssk); 1855 WRITE_ONCE(ssk->sk_rcvbuf, rcvbuf); 1856 tcp_sk(ssk)->window_clamp = window_clamp; 1857 tcp_cleanup_rbuf(ssk, 1); 1858 unlock_sock_fast(ssk, slow); 1859 } 1860 } 1861 } 1862 1863 msk->rcvq_space.space = msk->rcvq_space.copied; 1864 new_measure: 1865 msk->rcvq_space.copied = 0; 1866 msk->rcvq_space.time = mstamp; 1867 } 1868 1869 static void __mptcp_update_rmem(struct sock *sk) 1870 { 1871 struct mptcp_sock *msk = mptcp_sk(sk); 1872 1873 if (!msk->rmem_released) 1874 return; 1875 1876 atomic_sub(msk->rmem_released, &sk->sk_rmem_alloc); 1877 sk_mem_uncharge(sk, msk->rmem_released); 1878 WRITE_ONCE(msk->rmem_released, 0); 1879 } 1880 1881 static void __mptcp_splice_receive_queue(struct sock *sk) 1882 { 1883 struct mptcp_sock *msk = mptcp_sk(sk); 1884 1885 skb_queue_splice_tail_init(&sk->sk_receive_queue, &msk->receive_queue); 1886 } 1887 1888 static bool __mptcp_move_skbs(struct mptcp_sock *msk) 1889 { 1890 struct sock *sk = (struct sock *)msk; 1891 unsigned int moved = 0; 1892 bool ret, done; 1893 1894 mptcp_flush_join_list(msk); 1895 do { 1896 struct sock *ssk = mptcp_subflow_recv_lookup(msk); 1897 bool slowpath; 1898 1899 /* we can have data pending in the subflows only if the msk 1900 * receive buffer was full at subflow_data_ready() time, 1901 * that is an unlikely slow path. 1902 */ 1903 if (likely(!ssk)) 1904 break; 1905 1906 slowpath = lock_sock_fast(ssk); 1907 mptcp_data_lock(sk); 1908 __mptcp_update_rmem(sk); 1909 done = __mptcp_move_skbs_from_subflow(msk, ssk, &moved); 1910 mptcp_data_unlock(sk); 1911 1912 if (unlikely(ssk->sk_err)) 1913 __mptcp_error_report(sk); 1914 unlock_sock_fast(ssk, slowpath); 1915 } while (!done); 1916 1917 /* acquire the data lock only if some input data is pending */ 1918 ret = moved > 0; 1919 if (!RB_EMPTY_ROOT(&msk->out_of_order_queue) || 1920 !skb_queue_empty_lockless(&sk->sk_receive_queue)) { 1921 mptcp_data_lock(sk); 1922 __mptcp_update_rmem(sk); 1923 ret |= __mptcp_ofo_queue(msk); 1924 __mptcp_splice_receive_queue(sk); 1925 mptcp_data_unlock(sk); 1926 } 1927 if (ret) 1928 mptcp_check_data_fin((struct sock *)msk); 1929 return !skb_queue_empty(&msk->receive_queue); 1930 } 1931 1932 static int mptcp_recvmsg(struct sock *sk, struct msghdr *msg, size_t len, 1933 int nonblock, int flags, int *addr_len) 1934 { 1935 struct mptcp_sock *msk = mptcp_sk(sk); 1936 struct scm_timestamping_internal tss; 1937 int copied = 0, cmsg_flags = 0; 1938 int target; 1939 long timeo; 1940 1941 /* MSG_ERRQUEUE is really a no-op till we support IP_RECVERR */ 1942 if (unlikely(flags & MSG_ERRQUEUE)) 1943 return inet_recv_error(sk, msg, len, addr_len); 1944 1945 mptcp_lock_sock(sk, __mptcp_splice_receive_queue(sk)); 1946 if (unlikely(sk->sk_state == TCP_LISTEN)) { 1947 copied = -ENOTCONN; 1948 goto out_err; 1949 } 1950 1951 timeo = sock_rcvtimeo(sk, nonblock); 1952 1953 len = min_t(size_t, len, INT_MAX); 1954 target = sock_rcvlowat(sk, flags & MSG_WAITALL, len); 1955 1956 while (copied < len) { 1957 int bytes_read; 1958 1959 bytes_read = __mptcp_recvmsg_mskq(msk, msg, len - copied, flags, &tss, &cmsg_flags); 1960 if (unlikely(bytes_read < 0)) { 1961 if (!copied) 1962 copied = bytes_read; 1963 goto out_err; 1964 } 1965 1966 copied += bytes_read; 1967 1968 /* be sure to advertise window change */ 1969 mptcp_cleanup_rbuf(msk); 1970 1971 if (skb_queue_empty(&msk->receive_queue) && __mptcp_move_skbs(msk)) 1972 continue; 1973 1974 /* only the master socket status is relevant here. The exit 1975 * conditions mirror closely tcp_recvmsg() 1976 */ 1977 if (copied >= target) 1978 break; 1979 1980 if (copied) { 1981 if (sk->sk_err || 1982 sk->sk_state == TCP_CLOSE || 1983 (sk->sk_shutdown & RCV_SHUTDOWN) || 1984 !timeo || 1985 signal_pending(current)) 1986 break; 1987 } else { 1988 if (sk->sk_err) { 1989 copied = sock_error(sk); 1990 break; 1991 } 1992 1993 if (test_and_clear_bit(MPTCP_WORK_EOF, &msk->flags)) 1994 mptcp_check_for_eof(msk); 1995 1996 if (sk->sk_shutdown & RCV_SHUTDOWN) { 1997 /* race breaker: the shutdown could be after the 1998 * previous receive queue check 1999 */ 2000 if (__mptcp_move_skbs(msk)) 2001 continue; 2002 break; 2003 } 2004 2005 if (sk->sk_state == TCP_CLOSE) { 2006 copied = -ENOTCONN; 2007 break; 2008 } 2009 2010 if (!timeo) { 2011 copied = -EAGAIN; 2012 break; 2013 } 2014 2015 if (signal_pending(current)) { 2016 copied = sock_intr_errno(timeo); 2017 break; 2018 } 2019 } 2020 2021 pr_debug("block timeout %ld", timeo); 2022 mptcp_wait_data(sk, &timeo); 2023 } 2024 2025 if (skb_queue_empty_lockless(&sk->sk_receive_queue) && 2026 skb_queue_empty(&msk->receive_queue)) { 2027 /* entire backlog drained, clear DATA_READY. */ 2028 clear_bit(MPTCP_DATA_READY, &msk->flags); 2029 2030 /* .. race-breaker: ssk might have gotten new data 2031 * after last __mptcp_move_skbs() returned false. 2032 */ 2033 if (unlikely(__mptcp_move_skbs(msk))) 2034 set_bit(MPTCP_DATA_READY, &msk->flags); 2035 } 2036 2037 out_err: 2038 if (cmsg_flags && copied >= 0) { 2039 if (cmsg_flags & MPTCP_CMSG_TS) 2040 tcp_recv_timestamp(msg, sk, &tss); 2041 } 2042 2043 pr_debug("msk=%p data_ready=%d rx queue empty=%d copied=%d", 2044 msk, test_bit(MPTCP_DATA_READY, &msk->flags), 2045 skb_queue_empty_lockless(&sk->sk_receive_queue), copied); 2046 if (!(flags & MSG_PEEK)) 2047 mptcp_rcv_space_adjust(msk, copied); 2048 2049 release_sock(sk); 2050 return copied; 2051 } 2052 2053 static void mptcp_retransmit_timer(struct timer_list *t) 2054 { 2055 struct inet_connection_sock *icsk = from_timer(icsk, t, 2056 icsk_retransmit_timer); 2057 struct sock *sk = &icsk->icsk_inet.sk; 2058 struct mptcp_sock *msk = mptcp_sk(sk); 2059 2060 bh_lock_sock(sk); 2061 if (!sock_owned_by_user(sk)) { 2062 /* we need a process context to retransmit */ 2063 if (!test_and_set_bit(MPTCP_WORK_RTX, &msk->flags)) 2064 mptcp_schedule_work(sk); 2065 } else { 2066 /* delegate our work to tcp_release_cb() */ 2067 set_bit(MPTCP_RETRANSMIT, &msk->flags); 2068 } 2069 bh_unlock_sock(sk); 2070 sock_put(sk); 2071 } 2072 2073 static void mptcp_timeout_timer(struct timer_list *t) 2074 { 2075 struct sock *sk = from_timer(sk, t, sk_timer); 2076 2077 mptcp_schedule_work(sk); 2078 sock_put(sk); 2079 } 2080 2081 /* Find an idle subflow. Return NULL if there is unacked data at tcp 2082 * level. 2083 * 2084 * A backup subflow is returned only if that is the only kind available. 2085 */ 2086 static struct sock *mptcp_subflow_get_retrans(const struct mptcp_sock *msk) 2087 { 2088 struct mptcp_subflow_context *subflow; 2089 struct sock *backup = NULL; 2090 2091 sock_owned_by_me((const struct sock *)msk); 2092 2093 if (__mptcp_check_fallback(msk)) 2094 return NULL; 2095 2096 mptcp_for_each_subflow(msk, subflow) { 2097 struct sock *ssk = mptcp_subflow_tcp_sock(subflow); 2098 2099 if (!mptcp_subflow_active(subflow)) 2100 continue; 2101 2102 /* still data outstanding at TCP level? Don't retransmit. */ 2103 if (!tcp_write_queue_empty(ssk)) { 2104 if (inet_csk(ssk)->icsk_ca_state >= TCP_CA_Loss) 2105 continue; 2106 return NULL; 2107 } 2108 2109 if (subflow->backup) { 2110 if (!backup) 2111 backup = ssk; 2112 continue; 2113 } 2114 2115 return ssk; 2116 } 2117 2118 return backup; 2119 } 2120 2121 static void mptcp_dispose_initial_subflow(struct mptcp_sock *msk) 2122 { 2123 if (msk->subflow) { 2124 iput(SOCK_INODE(msk->subflow)); 2125 msk->subflow = NULL; 2126 } 2127 } 2128 2129 /* subflow sockets can be either outgoing (connect) or incoming 2130 * (accept). 2131 * 2132 * Outgoing subflows use in-kernel sockets. 2133 * Incoming subflows do not have their own 'struct socket' allocated, 2134 * so we need to use tcp_close() after detaching them from the mptcp 2135 * parent socket. 2136 */ 2137 static void __mptcp_close_ssk(struct sock *sk, struct sock *ssk, 2138 struct mptcp_subflow_context *subflow) 2139 { 2140 struct mptcp_sock *msk = mptcp_sk(sk); 2141 2142 list_del(&subflow->node); 2143 2144 lock_sock_nested(ssk, SINGLE_DEPTH_NESTING); 2145 2146 /* if we are invoked by the msk cleanup code, the subflow is 2147 * already orphaned 2148 */ 2149 if (ssk->sk_socket) 2150 sock_orphan(ssk); 2151 2152 subflow->disposable = 1; 2153 2154 /* if ssk hit tcp_done(), tcp_cleanup_ulp() cleared the related ops 2155 * the ssk has been already destroyed, we just need to release the 2156 * reference owned by msk; 2157 */ 2158 if (!inet_csk(ssk)->icsk_ulp_ops) { 2159 kfree_rcu(subflow, rcu); 2160 } else { 2161 /* otherwise tcp will dispose of the ssk and subflow ctx */ 2162 __tcp_close(ssk, 0); 2163 2164 /* close acquired an extra ref */ 2165 __sock_put(ssk); 2166 } 2167 release_sock(ssk); 2168 2169 sock_put(ssk); 2170 2171 if (ssk == msk->last_snd) 2172 msk->last_snd = NULL; 2173 2174 if (ssk == msk->first) 2175 msk->first = NULL; 2176 2177 if (msk->subflow && ssk == msk->subflow->sk) 2178 mptcp_dispose_initial_subflow(msk); 2179 } 2180 2181 void mptcp_close_ssk(struct sock *sk, struct sock *ssk, 2182 struct mptcp_subflow_context *subflow) 2183 { 2184 if (sk->sk_state == TCP_ESTABLISHED) 2185 mptcp_event(MPTCP_EVENT_SUB_CLOSED, mptcp_sk(sk), ssk, GFP_KERNEL); 2186 __mptcp_close_ssk(sk, ssk, subflow); 2187 } 2188 2189 static unsigned int mptcp_sync_mss(struct sock *sk, u32 pmtu) 2190 { 2191 return 0; 2192 } 2193 2194 static void __mptcp_close_subflow(struct mptcp_sock *msk) 2195 { 2196 struct mptcp_subflow_context *subflow, *tmp; 2197 2198 might_sleep(); 2199 2200 list_for_each_entry_safe(subflow, tmp, &msk->conn_list, node) { 2201 struct sock *ssk = mptcp_subflow_tcp_sock(subflow); 2202 2203 if (inet_sk_state_load(ssk) != TCP_CLOSE) 2204 continue; 2205 2206 /* 'subflow_data_ready' will re-sched once rx queue is empty */ 2207 if (!skb_queue_empty_lockless(&ssk->sk_receive_queue)) 2208 continue; 2209 2210 mptcp_close_ssk((struct sock *)msk, ssk, subflow); 2211 } 2212 } 2213 2214 static bool mptcp_check_close_timeout(const struct sock *sk) 2215 { 2216 s32 delta = tcp_jiffies32 - inet_csk(sk)->icsk_mtup.probe_timestamp; 2217 struct mptcp_subflow_context *subflow; 2218 2219 if (delta >= TCP_TIMEWAIT_LEN) 2220 return true; 2221 2222 /* if all subflows are in closed status don't bother with additional 2223 * timeout 2224 */ 2225 mptcp_for_each_subflow(mptcp_sk(sk), subflow) { 2226 if (inet_sk_state_load(mptcp_subflow_tcp_sock(subflow)) != 2227 TCP_CLOSE) 2228 return false; 2229 } 2230 return true; 2231 } 2232 2233 static void mptcp_check_fastclose(struct mptcp_sock *msk) 2234 { 2235 struct mptcp_subflow_context *subflow, *tmp; 2236 struct sock *sk = &msk->sk.icsk_inet.sk; 2237 2238 if (likely(!READ_ONCE(msk->rcv_fastclose))) 2239 return; 2240 2241 mptcp_token_destroy(msk); 2242 2243 list_for_each_entry_safe(subflow, tmp, &msk->conn_list, node) { 2244 struct sock *tcp_sk = mptcp_subflow_tcp_sock(subflow); 2245 bool slow; 2246 2247 slow = lock_sock_fast(tcp_sk); 2248 if (tcp_sk->sk_state != TCP_CLOSE) { 2249 tcp_send_active_reset(tcp_sk, GFP_ATOMIC); 2250 tcp_set_state(tcp_sk, TCP_CLOSE); 2251 } 2252 unlock_sock_fast(tcp_sk, slow); 2253 } 2254 2255 inet_sk_state_store(sk, TCP_CLOSE); 2256 sk->sk_shutdown = SHUTDOWN_MASK; 2257 smp_mb__before_atomic(); /* SHUTDOWN must be visible first */ 2258 set_bit(MPTCP_DATA_READY, &msk->flags); 2259 set_bit(MPTCP_WORK_CLOSE_SUBFLOW, &msk->flags); 2260 2261 mptcp_close_wake_up(sk); 2262 } 2263 2264 static void __mptcp_retrans(struct sock *sk) 2265 { 2266 struct mptcp_sock *msk = mptcp_sk(sk); 2267 struct mptcp_sendmsg_info info = {}; 2268 struct mptcp_data_frag *dfrag; 2269 size_t copied = 0; 2270 struct sock *ssk; 2271 int ret; 2272 2273 mptcp_clean_una_wakeup(sk); 2274 dfrag = mptcp_rtx_head(sk); 2275 if (!dfrag) { 2276 if (mptcp_data_fin_enabled(msk)) { 2277 struct inet_connection_sock *icsk = inet_csk(sk); 2278 2279 icsk->icsk_retransmits++; 2280 mptcp_set_datafin_timeout(sk); 2281 mptcp_send_ack(msk); 2282 2283 goto reset_timer; 2284 } 2285 2286 return; 2287 } 2288 2289 ssk = mptcp_subflow_get_retrans(msk); 2290 if (!ssk) 2291 goto reset_timer; 2292 2293 lock_sock(ssk); 2294 2295 /* limit retransmission to the bytes already sent on some subflows */ 2296 info.sent = 0; 2297 info.limit = READ_ONCE(msk->csum_enabled) ? dfrag->data_len : dfrag->already_sent; 2298 while (info.sent < info.limit) { 2299 if (!mptcp_alloc_tx_skb(sk, ssk)) 2300 break; 2301 2302 ret = mptcp_sendmsg_frag(sk, ssk, dfrag, &info); 2303 if (ret <= 0) 2304 break; 2305 2306 MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_RETRANSSEGS); 2307 copied += ret; 2308 info.sent += ret; 2309 } 2310 if (copied) { 2311 dfrag->already_sent = max(dfrag->already_sent, info.sent); 2312 tcp_push(ssk, 0, info.mss_now, tcp_sk(ssk)->nonagle, 2313 info.size_goal); 2314 } 2315 2316 mptcp_set_timeout(sk, ssk); 2317 release_sock(ssk); 2318 2319 reset_timer: 2320 if (!mptcp_timer_pending(sk)) 2321 mptcp_reset_timer(sk); 2322 } 2323 2324 static void mptcp_worker(struct work_struct *work) 2325 { 2326 struct mptcp_sock *msk = container_of(work, struct mptcp_sock, work); 2327 struct sock *sk = &msk->sk.icsk_inet.sk; 2328 int state; 2329 2330 lock_sock(sk); 2331 state = sk->sk_state; 2332 if (unlikely(state == TCP_CLOSE)) 2333 goto unlock; 2334 2335 mptcp_check_data_fin_ack(sk); 2336 mptcp_flush_join_list(msk); 2337 2338 mptcp_check_fastclose(msk); 2339 2340 if (msk->pm.status) 2341 mptcp_pm_nl_work(msk); 2342 2343 if (test_and_clear_bit(MPTCP_WORK_EOF, &msk->flags)) 2344 mptcp_check_for_eof(msk); 2345 2346 __mptcp_check_send_data_fin(sk); 2347 mptcp_check_data_fin(sk); 2348 2349 /* There is no point in keeping around an orphaned sk timedout or 2350 * closed, but we need the msk around to reply to incoming DATA_FIN, 2351 * even if it is orphaned and in FIN_WAIT2 state 2352 */ 2353 if (sock_flag(sk, SOCK_DEAD) && 2354 (mptcp_check_close_timeout(sk) || sk->sk_state == TCP_CLOSE)) { 2355 inet_sk_state_store(sk, TCP_CLOSE); 2356 __mptcp_destroy_sock(sk); 2357 goto unlock; 2358 } 2359 2360 if (test_and_clear_bit(MPTCP_WORK_CLOSE_SUBFLOW, &msk->flags)) 2361 __mptcp_close_subflow(msk); 2362 2363 if (test_and_clear_bit(MPTCP_WORK_RTX, &msk->flags)) 2364 __mptcp_retrans(sk); 2365 2366 unlock: 2367 release_sock(sk); 2368 sock_put(sk); 2369 } 2370 2371 static int __mptcp_init_sock(struct sock *sk) 2372 { 2373 struct mptcp_sock *msk = mptcp_sk(sk); 2374 2375 spin_lock_init(&msk->join_list_lock); 2376 2377 INIT_LIST_HEAD(&msk->conn_list); 2378 INIT_LIST_HEAD(&msk->join_list); 2379 INIT_LIST_HEAD(&msk->rtx_queue); 2380 INIT_WORK(&msk->work, mptcp_worker); 2381 __skb_queue_head_init(&msk->receive_queue); 2382 msk->out_of_order_queue = RB_ROOT; 2383 msk->first_pending = NULL; 2384 msk->wmem_reserved = 0; 2385 WRITE_ONCE(msk->rmem_released, 0); 2386 msk->tx_pending_data = 0; 2387 2388 msk->first = NULL; 2389 inet_csk(sk)->icsk_sync_mss = mptcp_sync_mss; 2390 WRITE_ONCE(msk->csum_enabled, mptcp_is_checksum_enabled(sock_net(sk))); 2391 2392 mptcp_pm_data_init(msk); 2393 2394 /* re-use the csk retrans timer for MPTCP-level retrans */ 2395 timer_setup(&msk->sk.icsk_retransmit_timer, mptcp_retransmit_timer, 0); 2396 timer_setup(&sk->sk_timer, mptcp_timeout_timer, 0); 2397 2398 return 0; 2399 } 2400 2401 static int mptcp_init_sock(struct sock *sk) 2402 { 2403 struct inet_connection_sock *icsk = inet_csk(sk); 2404 struct net *net = sock_net(sk); 2405 int ret; 2406 2407 ret = __mptcp_init_sock(sk); 2408 if (ret) 2409 return ret; 2410 2411 if (!mptcp_is_enabled(net)) 2412 return -ENOPROTOOPT; 2413 2414 if (unlikely(!net->mib.mptcp_statistics) && !mptcp_mib_alloc(net)) 2415 return -ENOMEM; 2416 2417 ret = __mptcp_socket_create(mptcp_sk(sk)); 2418 if (ret) 2419 return ret; 2420 2421 /* fetch the ca name; do it outside __mptcp_init_sock(), so that clone will 2422 * propagate the correct value 2423 */ 2424 tcp_assign_congestion_control(sk); 2425 strcpy(mptcp_sk(sk)->ca_name, icsk->icsk_ca_ops->name); 2426 2427 /* no need to keep a reference to the ops, the name will suffice */ 2428 tcp_cleanup_congestion_control(sk); 2429 icsk->icsk_ca_ops = NULL; 2430 2431 sk_sockets_allocated_inc(sk); 2432 sk->sk_rcvbuf = sock_net(sk)->ipv4.sysctl_tcp_rmem[1]; 2433 sk->sk_sndbuf = sock_net(sk)->ipv4.sysctl_tcp_wmem[1]; 2434 2435 return 0; 2436 } 2437 2438 static void __mptcp_clear_xmit(struct sock *sk) 2439 { 2440 struct mptcp_sock *msk = mptcp_sk(sk); 2441 struct mptcp_data_frag *dtmp, *dfrag; 2442 2443 WRITE_ONCE(msk->first_pending, NULL); 2444 list_for_each_entry_safe(dfrag, dtmp, &msk->rtx_queue, list) 2445 dfrag_clear(sk, dfrag); 2446 } 2447 2448 static void mptcp_cancel_work(struct sock *sk) 2449 { 2450 struct mptcp_sock *msk = mptcp_sk(sk); 2451 2452 if (cancel_work_sync(&msk->work)) 2453 __sock_put(sk); 2454 } 2455 2456 void mptcp_subflow_shutdown(struct sock *sk, struct sock *ssk, int how) 2457 { 2458 lock_sock(ssk); 2459 2460 switch (ssk->sk_state) { 2461 case TCP_LISTEN: 2462 if (!(how & RCV_SHUTDOWN)) 2463 break; 2464 fallthrough; 2465 case TCP_SYN_SENT: 2466 tcp_disconnect(ssk, O_NONBLOCK); 2467 break; 2468 default: 2469 if (__mptcp_check_fallback(mptcp_sk(sk))) { 2470 pr_debug("Fallback"); 2471 ssk->sk_shutdown |= how; 2472 tcp_shutdown(ssk, how); 2473 } else { 2474 pr_debug("Sending DATA_FIN on subflow %p", ssk); 2475 mptcp_set_timeout(sk, ssk); 2476 tcp_send_ack(ssk); 2477 if (!mptcp_timer_pending(sk)) 2478 mptcp_reset_timer(sk); 2479 } 2480 break; 2481 } 2482 2483 release_sock(ssk); 2484 } 2485 2486 static const unsigned char new_state[16] = { 2487 /* current state: new state: action: */ 2488 [0 /* (Invalid) */] = TCP_CLOSE, 2489 [TCP_ESTABLISHED] = TCP_FIN_WAIT1 | TCP_ACTION_FIN, 2490 [TCP_SYN_SENT] = TCP_CLOSE, 2491 [TCP_SYN_RECV] = TCP_FIN_WAIT1 | TCP_ACTION_FIN, 2492 [TCP_FIN_WAIT1] = TCP_FIN_WAIT1, 2493 [TCP_FIN_WAIT2] = TCP_FIN_WAIT2, 2494 [TCP_TIME_WAIT] = TCP_CLOSE, /* should not happen ! */ 2495 [TCP_CLOSE] = TCP_CLOSE, 2496 [TCP_CLOSE_WAIT] = TCP_LAST_ACK | TCP_ACTION_FIN, 2497 [TCP_LAST_ACK] = TCP_LAST_ACK, 2498 [TCP_LISTEN] = TCP_CLOSE, 2499 [TCP_CLOSING] = TCP_CLOSING, 2500 [TCP_NEW_SYN_RECV] = TCP_CLOSE, /* should not happen ! */ 2501 }; 2502 2503 static int mptcp_close_state(struct sock *sk) 2504 { 2505 int next = (int)new_state[sk->sk_state]; 2506 int ns = next & TCP_STATE_MASK; 2507 2508 inet_sk_state_store(sk, ns); 2509 2510 return next & TCP_ACTION_FIN; 2511 } 2512 2513 static void __mptcp_check_send_data_fin(struct sock *sk) 2514 { 2515 struct mptcp_subflow_context *subflow; 2516 struct mptcp_sock *msk = mptcp_sk(sk); 2517 2518 pr_debug("msk=%p snd_data_fin_enable=%d pending=%d snd_nxt=%llu write_seq=%llu", 2519 msk, msk->snd_data_fin_enable, !!mptcp_send_head(sk), 2520 msk->snd_nxt, msk->write_seq); 2521 2522 /* we still need to enqueue subflows or not really shutting down, 2523 * skip this 2524 */ 2525 if (!msk->snd_data_fin_enable || msk->snd_nxt + 1 != msk->write_seq || 2526 mptcp_send_head(sk)) 2527 return; 2528 2529 WRITE_ONCE(msk->snd_nxt, msk->write_seq); 2530 2531 /* fallback socket will not get data_fin/ack, can move to the next 2532 * state now 2533 */ 2534 if (__mptcp_check_fallback(msk)) { 2535 if ((1 << sk->sk_state) & (TCPF_CLOSING | TCPF_LAST_ACK)) { 2536 inet_sk_state_store(sk, TCP_CLOSE); 2537 mptcp_close_wake_up(sk); 2538 } else if (sk->sk_state == TCP_FIN_WAIT1) { 2539 inet_sk_state_store(sk, TCP_FIN_WAIT2); 2540 } 2541 } 2542 2543 mptcp_flush_join_list(msk); 2544 mptcp_for_each_subflow(msk, subflow) { 2545 struct sock *tcp_sk = mptcp_subflow_tcp_sock(subflow); 2546 2547 mptcp_subflow_shutdown(sk, tcp_sk, SEND_SHUTDOWN); 2548 } 2549 } 2550 2551 static void __mptcp_wr_shutdown(struct sock *sk) 2552 { 2553 struct mptcp_sock *msk = mptcp_sk(sk); 2554 2555 pr_debug("msk=%p snd_data_fin_enable=%d shutdown=%x state=%d pending=%d", 2556 msk, msk->snd_data_fin_enable, sk->sk_shutdown, sk->sk_state, 2557 !!mptcp_send_head(sk)); 2558 2559 /* will be ignored by fallback sockets */ 2560 WRITE_ONCE(msk->write_seq, msk->write_seq + 1); 2561 WRITE_ONCE(msk->snd_data_fin_enable, 1); 2562 2563 __mptcp_check_send_data_fin(sk); 2564 } 2565 2566 static void __mptcp_destroy_sock(struct sock *sk) 2567 { 2568 struct mptcp_subflow_context *subflow, *tmp; 2569 struct mptcp_sock *msk = mptcp_sk(sk); 2570 LIST_HEAD(conn_list); 2571 2572 pr_debug("msk=%p", msk); 2573 2574 might_sleep(); 2575 2576 /* be sure to always acquire the join list lock, to sync vs 2577 * mptcp_finish_join(). 2578 */ 2579 spin_lock_bh(&msk->join_list_lock); 2580 list_splice_tail_init(&msk->join_list, &msk->conn_list); 2581 spin_unlock_bh(&msk->join_list_lock); 2582 list_splice_init(&msk->conn_list, &conn_list); 2583 2584 sk_stop_timer(sk, &msk->sk.icsk_retransmit_timer); 2585 sk_stop_timer(sk, &sk->sk_timer); 2586 msk->pm.status = 0; 2587 2588 list_for_each_entry_safe(subflow, tmp, &conn_list, node) { 2589 struct sock *ssk = mptcp_subflow_tcp_sock(subflow); 2590 __mptcp_close_ssk(sk, ssk, subflow); 2591 } 2592 2593 sk->sk_prot->destroy(sk); 2594 2595 WARN_ON_ONCE(msk->wmem_reserved); 2596 WARN_ON_ONCE(msk->rmem_released); 2597 sk_stream_kill_queues(sk); 2598 xfrm_sk_free_policy(sk); 2599 2600 sk_refcnt_debug_release(sk); 2601 mptcp_dispose_initial_subflow(msk); 2602 sock_put(sk); 2603 } 2604 2605 static void mptcp_close(struct sock *sk, long timeout) 2606 { 2607 struct mptcp_subflow_context *subflow; 2608 bool do_cancel_work = false; 2609 2610 lock_sock(sk); 2611 sk->sk_shutdown = SHUTDOWN_MASK; 2612 2613 if ((1 << sk->sk_state) & (TCPF_LISTEN | TCPF_CLOSE)) { 2614 inet_sk_state_store(sk, TCP_CLOSE); 2615 goto cleanup; 2616 } 2617 2618 if (mptcp_close_state(sk)) 2619 __mptcp_wr_shutdown(sk); 2620 2621 sk_stream_wait_close(sk, timeout); 2622 2623 cleanup: 2624 /* orphan all the subflows */ 2625 inet_csk(sk)->icsk_mtup.probe_timestamp = tcp_jiffies32; 2626 mptcp_for_each_subflow(mptcp_sk(sk), subflow) { 2627 struct sock *ssk = mptcp_subflow_tcp_sock(subflow); 2628 bool slow = lock_sock_fast(ssk); 2629 2630 sock_orphan(ssk); 2631 unlock_sock_fast(ssk, slow); 2632 } 2633 sock_orphan(sk); 2634 2635 sock_hold(sk); 2636 pr_debug("msk=%p state=%d", sk, sk->sk_state); 2637 if (sk->sk_state == TCP_CLOSE) { 2638 __mptcp_destroy_sock(sk); 2639 do_cancel_work = true; 2640 } else { 2641 sk_reset_timer(sk, &sk->sk_timer, jiffies + TCP_TIMEWAIT_LEN); 2642 } 2643 release_sock(sk); 2644 if (do_cancel_work) 2645 mptcp_cancel_work(sk); 2646 2647 if (mptcp_sk(sk)->token) 2648 mptcp_event(MPTCP_EVENT_CLOSED, mptcp_sk(sk), NULL, GFP_KERNEL); 2649 2650 sock_put(sk); 2651 } 2652 2653 static void mptcp_copy_inaddrs(struct sock *msk, const struct sock *ssk) 2654 { 2655 #if IS_ENABLED(CONFIG_MPTCP_IPV6) 2656 const struct ipv6_pinfo *ssk6 = inet6_sk(ssk); 2657 struct ipv6_pinfo *msk6 = inet6_sk(msk); 2658 2659 msk->sk_v6_daddr = ssk->sk_v6_daddr; 2660 msk->sk_v6_rcv_saddr = ssk->sk_v6_rcv_saddr; 2661 2662 if (msk6 && ssk6) { 2663 msk6->saddr = ssk6->saddr; 2664 msk6->flow_label = ssk6->flow_label; 2665 } 2666 #endif 2667 2668 inet_sk(msk)->inet_num = inet_sk(ssk)->inet_num; 2669 inet_sk(msk)->inet_dport = inet_sk(ssk)->inet_dport; 2670 inet_sk(msk)->inet_sport = inet_sk(ssk)->inet_sport; 2671 inet_sk(msk)->inet_daddr = inet_sk(ssk)->inet_daddr; 2672 inet_sk(msk)->inet_saddr = inet_sk(ssk)->inet_saddr; 2673 inet_sk(msk)->inet_rcv_saddr = inet_sk(ssk)->inet_rcv_saddr; 2674 } 2675 2676 static int mptcp_disconnect(struct sock *sk, int flags) 2677 { 2678 struct mptcp_subflow_context *subflow; 2679 struct mptcp_sock *msk = mptcp_sk(sk); 2680 2681 mptcp_do_flush_join_list(msk); 2682 2683 mptcp_for_each_subflow(msk, subflow) { 2684 struct sock *ssk = mptcp_subflow_tcp_sock(subflow); 2685 2686 lock_sock(ssk); 2687 tcp_disconnect(ssk, flags); 2688 release_sock(ssk); 2689 } 2690 return 0; 2691 } 2692 2693 #if IS_ENABLED(CONFIG_MPTCP_IPV6) 2694 static struct ipv6_pinfo *mptcp_inet6_sk(const struct sock *sk) 2695 { 2696 unsigned int offset = sizeof(struct mptcp6_sock) - sizeof(struct ipv6_pinfo); 2697 2698 return (struct ipv6_pinfo *)(((u8 *)sk) + offset); 2699 } 2700 #endif 2701 2702 struct sock *mptcp_sk_clone(const struct sock *sk, 2703 const struct mptcp_options_received *mp_opt, 2704 struct request_sock *req) 2705 { 2706 struct mptcp_subflow_request_sock *subflow_req = mptcp_subflow_rsk(req); 2707 struct sock *nsk = sk_clone_lock(sk, GFP_ATOMIC); 2708 struct mptcp_sock *msk; 2709 u64 ack_seq; 2710 2711 if (!nsk) 2712 return NULL; 2713 2714 #if IS_ENABLED(CONFIG_MPTCP_IPV6) 2715 if (nsk->sk_family == AF_INET6) 2716 inet_sk(nsk)->pinet6 = mptcp_inet6_sk(nsk); 2717 #endif 2718 2719 __mptcp_init_sock(nsk); 2720 2721 msk = mptcp_sk(nsk); 2722 msk->local_key = subflow_req->local_key; 2723 msk->token = subflow_req->token; 2724 msk->subflow = NULL; 2725 WRITE_ONCE(msk->fully_established, false); 2726 if (mp_opt->csum_reqd) 2727 WRITE_ONCE(msk->csum_enabled, true); 2728 2729 msk->write_seq = subflow_req->idsn + 1; 2730 msk->snd_nxt = msk->write_seq; 2731 msk->snd_una = msk->write_seq; 2732 msk->wnd_end = msk->snd_nxt + req->rsk_rcv_wnd; 2733 msk->setsockopt_seq = mptcp_sk(sk)->setsockopt_seq; 2734 2735 if (mp_opt->mp_capable) { 2736 msk->can_ack = true; 2737 msk->remote_key = mp_opt->sndr_key; 2738 mptcp_crypto_key_sha(msk->remote_key, NULL, &ack_seq); 2739 ack_seq++; 2740 WRITE_ONCE(msk->ack_seq, ack_seq); 2741 WRITE_ONCE(msk->rcv_wnd_sent, ack_seq); 2742 } 2743 2744 sock_reset_flag(nsk, SOCK_RCU_FREE); 2745 /* will be fully established after successful MPC subflow creation */ 2746 inet_sk_state_store(nsk, TCP_SYN_RECV); 2747 2748 security_inet_csk_clone(nsk, req); 2749 bh_unlock_sock(nsk); 2750 2751 /* keep a single reference */ 2752 __sock_put(nsk); 2753 return nsk; 2754 } 2755 2756 void mptcp_rcv_space_init(struct mptcp_sock *msk, const struct sock *ssk) 2757 { 2758 const struct tcp_sock *tp = tcp_sk(ssk); 2759 2760 msk->rcvq_space.copied = 0; 2761 msk->rcvq_space.rtt_us = 0; 2762 2763 msk->rcvq_space.time = tp->tcp_mstamp; 2764 2765 /* initial rcv_space offering made to peer */ 2766 msk->rcvq_space.space = min_t(u32, tp->rcv_wnd, 2767 TCP_INIT_CWND * tp->advmss); 2768 if (msk->rcvq_space.space == 0) 2769 msk->rcvq_space.space = TCP_INIT_CWND * TCP_MSS_DEFAULT; 2770 2771 WRITE_ONCE(msk->wnd_end, msk->snd_nxt + tcp_sk(ssk)->snd_wnd); 2772 } 2773 2774 static struct sock *mptcp_accept(struct sock *sk, int flags, int *err, 2775 bool kern) 2776 { 2777 struct mptcp_sock *msk = mptcp_sk(sk); 2778 struct socket *listener; 2779 struct sock *newsk; 2780 2781 listener = __mptcp_nmpc_socket(msk); 2782 if (WARN_ON_ONCE(!listener)) { 2783 *err = -EINVAL; 2784 return NULL; 2785 } 2786 2787 pr_debug("msk=%p, listener=%p", msk, mptcp_subflow_ctx(listener->sk)); 2788 newsk = inet_csk_accept(listener->sk, flags, err, kern); 2789 if (!newsk) 2790 return NULL; 2791 2792 pr_debug("msk=%p, subflow is mptcp=%d", msk, sk_is_mptcp(newsk)); 2793 if (sk_is_mptcp(newsk)) { 2794 struct mptcp_subflow_context *subflow; 2795 struct sock *new_mptcp_sock; 2796 2797 subflow = mptcp_subflow_ctx(newsk); 2798 new_mptcp_sock = subflow->conn; 2799 2800 /* is_mptcp should be false if subflow->conn is missing, see 2801 * subflow_syn_recv_sock() 2802 */ 2803 if (WARN_ON_ONCE(!new_mptcp_sock)) { 2804 tcp_sk(newsk)->is_mptcp = 0; 2805 return newsk; 2806 } 2807 2808 /* acquire the 2nd reference for the owning socket */ 2809 sock_hold(new_mptcp_sock); 2810 newsk = new_mptcp_sock; 2811 MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_MPCAPABLEPASSIVEACK); 2812 } else { 2813 MPTCP_INC_STATS(sock_net(sk), 2814 MPTCP_MIB_MPCAPABLEPASSIVEFALLBACK); 2815 } 2816 2817 return newsk; 2818 } 2819 2820 void mptcp_destroy_common(struct mptcp_sock *msk) 2821 { 2822 struct sock *sk = (struct sock *)msk; 2823 2824 __mptcp_clear_xmit(sk); 2825 2826 /* move to sk_receive_queue, sk_stream_kill_queues will purge it */ 2827 skb_queue_splice_tail_init(&msk->receive_queue, &sk->sk_receive_queue); 2828 2829 skb_rbtree_purge(&msk->out_of_order_queue); 2830 mptcp_token_destroy(msk); 2831 mptcp_pm_free_anno_list(msk); 2832 } 2833 2834 static void mptcp_destroy(struct sock *sk) 2835 { 2836 struct mptcp_sock *msk = mptcp_sk(sk); 2837 2838 mptcp_destroy_common(msk); 2839 sk_sockets_allocated_dec(sk); 2840 } 2841 2842 void __mptcp_data_acked(struct sock *sk) 2843 { 2844 if (!sock_owned_by_user(sk)) 2845 __mptcp_clean_una(sk); 2846 else 2847 set_bit(MPTCP_CLEAN_UNA, &mptcp_sk(sk)->flags); 2848 2849 if (mptcp_pending_data_fin_ack(sk)) 2850 mptcp_schedule_work(sk); 2851 } 2852 2853 void __mptcp_check_push(struct sock *sk, struct sock *ssk) 2854 { 2855 if (!mptcp_send_head(sk)) 2856 return; 2857 2858 if (!sock_owned_by_user(sk)) { 2859 struct sock *xmit_ssk = mptcp_subflow_get_send(mptcp_sk(sk)); 2860 2861 if (xmit_ssk == ssk) 2862 __mptcp_subflow_push_pending(sk, ssk); 2863 else if (xmit_ssk) 2864 mptcp_subflow_delegate(mptcp_subflow_ctx(xmit_ssk)); 2865 } else { 2866 set_bit(MPTCP_PUSH_PENDING, &mptcp_sk(sk)->flags); 2867 } 2868 } 2869 2870 /* processes deferred events and flush wmem */ 2871 static void mptcp_release_cb(struct sock *sk) 2872 { 2873 for (;;) { 2874 unsigned long flags = 0; 2875 2876 if (test_and_clear_bit(MPTCP_PUSH_PENDING, &mptcp_sk(sk)->flags)) 2877 flags |= BIT(MPTCP_PUSH_PENDING); 2878 if (test_and_clear_bit(MPTCP_RETRANSMIT, &mptcp_sk(sk)->flags)) 2879 flags |= BIT(MPTCP_RETRANSMIT); 2880 if (!flags) 2881 break; 2882 2883 /* the following actions acquire the subflow socket lock 2884 * 2885 * 1) can't be invoked in atomic scope 2886 * 2) must avoid ABBA deadlock with msk socket spinlock: the RX 2887 * datapath acquires the msk socket spinlock while helding 2888 * the subflow socket lock 2889 */ 2890 2891 spin_unlock_bh(&sk->sk_lock.slock); 2892 if (flags & BIT(MPTCP_PUSH_PENDING)) 2893 __mptcp_push_pending(sk, 0); 2894 if (flags & BIT(MPTCP_RETRANSMIT)) 2895 __mptcp_retrans(sk); 2896 2897 cond_resched(); 2898 spin_lock_bh(&sk->sk_lock.slock); 2899 } 2900 2901 /* be sure to set the current sk state before tacking actions 2902 * depending on sk_state 2903 */ 2904 if (test_and_clear_bit(MPTCP_CONNECTED, &mptcp_sk(sk)->flags)) 2905 __mptcp_set_connected(sk); 2906 if (test_and_clear_bit(MPTCP_CLEAN_UNA, &mptcp_sk(sk)->flags)) 2907 __mptcp_clean_una_wakeup(sk); 2908 if (test_and_clear_bit(MPTCP_ERROR_REPORT, &mptcp_sk(sk)->flags)) 2909 __mptcp_error_report(sk); 2910 2911 /* push_pending may touch wmem_reserved, ensure we do the cleanup 2912 * later 2913 */ 2914 __mptcp_update_wmem(sk); 2915 __mptcp_update_rmem(sk); 2916 } 2917 2918 void mptcp_subflow_process_delegated(struct sock *ssk) 2919 { 2920 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk); 2921 struct sock *sk = subflow->conn; 2922 2923 mptcp_data_lock(sk); 2924 if (!sock_owned_by_user(sk)) 2925 __mptcp_subflow_push_pending(sk, ssk); 2926 else 2927 set_bit(MPTCP_PUSH_PENDING, &mptcp_sk(sk)->flags); 2928 mptcp_data_unlock(sk); 2929 mptcp_subflow_delegated_done(subflow); 2930 } 2931 2932 static int mptcp_hash(struct sock *sk) 2933 { 2934 /* should never be called, 2935 * we hash the TCP subflows not the master socket 2936 */ 2937 WARN_ON_ONCE(1); 2938 return 0; 2939 } 2940 2941 static void mptcp_unhash(struct sock *sk) 2942 { 2943 /* called from sk_common_release(), but nothing to do here */ 2944 } 2945 2946 static int mptcp_get_port(struct sock *sk, unsigned short snum) 2947 { 2948 struct mptcp_sock *msk = mptcp_sk(sk); 2949 struct socket *ssock; 2950 2951 ssock = __mptcp_nmpc_socket(msk); 2952 pr_debug("msk=%p, subflow=%p", msk, ssock); 2953 if (WARN_ON_ONCE(!ssock)) 2954 return -EINVAL; 2955 2956 return inet_csk_get_port(ssock->sk, snum); 2957 } 2958 2959 void mptcp_finish_connect(struct sock *ssk) 2960 { 2961 struct mptcp_subflow_context *subflow; 2962 struct mptcp_sock *msk; 2963 struct sock *sk; 2964 u64 ack_seq; 2965 2966 subflow = mptcp_subflow_ctx(ssk); 2967 sk = subflow->conn; 2968 msk = mptcp_sk(sk); 2969 2970 pr_debug("msk=%p, token=%u", sk, subflow->token); 2971 2972 mptcp_crypto_key_sha(subflow->remote_key, NULL, &ack_seq); 2973 ack_seq++; 2974 subflow->map_seq = ack_seq; 2975 subflow->map_subflow_seq = 1; 2976 2977 /* the socket is not connected yet, no msk/subflow ops can access/race 2978 * accessing the field below 2979 */ 2980 WRITE_ONCE(msk->remote_key, subflow->remote_key); 2981 WRITE_ONCE(msk->local_key, subflow->local_key); 2982 WRITE_ONCE(msk->write_seq, subflow->idsn + 1); 2983 WRITE_ONCE(msk->snd_nxt, msk->write_seq); 2984 WRITE_ONCE(msk->ack_seq, ack_seq); 2985 WRITE_ONCE(msk->rcv_wnd_sent, ack_seq); 2986 WRITE_ONCE(msk->can_ack, 1); 2987 WRITE_ONCE(msk->snd_una, msk->write_seq); 2988 2989 mptcp_pm_new_connection(msk, ssk, 0); 2990 2991 mptcp_rcv_space_init(msk, ssk); 2992 } 2993 2994 void mptcp_sock_graft(struct sock *sk, struct socket *parent) 2995 { 2996 write_lock_bh(&sk->sk_callback_lock); 2997 rcu_assign_pointer(sk->sk_wq, &parent->wq); 2998 sk_set_socket(sk, parent); 2999 sk->sk_uid = SOCK_INODE(parent)->i_uid; 3000 write_unlock_bh(&sk->sk_callback_lock); 3001 } 3002 3003 bool mptcp_finish_join(struct sock *ssk) 3004 { 3005 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk); 3006 struct mptcp_sock *msk = mptcp_sk(subflow->conn); 3007 struct sock *parent = (void *)msk; 3008 struct socket *parent_sock; 3009 bool ret; 3010 3011 pr_debug("msk=%p, subflow=%p", msk, subflow); 3012 3013 /* mptcp socket already closing? */ 3014 if (!mptcp_is_fully_established(parent)) { 3015 subflow->reset_reason = MPTCP_RST_EMPTCP; 3016 return false; 3017 } 3018 3019 if (!msk->pm.server_side) 3020 goto out; 3021 3022 if (!mptcp_pm_allow_new_subflow(msk)) { 3023 subflow->reset_reason = MPTCP_RST_EPROHIBIT; 3024 return false; 3025 } 3026 3027 /* active connections are already on conn_list, and we can't acquire 3028 * msk lock here. 3029 * use the join list lock as synchronization point and double-check 3030 * msk status to avoid racing with __mptcp_destroy_sock() 3031 */ 3032 spin_lock_bh(&msk->join_list_lock); 3033 ret = inet_sk_state_load(parent) == TCP_ESTABLISHED; 3034 if (ret && !WARN_ON_ONCE(!list_empty(&subflow->node))) { 3035 list_add_tail(&subflow->node, &msk->join_list); 3036 sock_hold(ssk); 3037 } 3038 spin_unlock_bh(&msk->join_list_lock); 3039 if (!ret) { 3040 subflow->reset_reason = MPTCP_RST_EPROHIBIT; 3041 return false; 3042 } 3043 3044 /* attach to msk socket only after we are sure he will deal with us 3045 * at close time 3046 */ 3047 parent_sock = READ_ONCE(parent->sk_socket); 3048 if (parent_sock && !ssk->sk_socket) 3049 mptcp_sock_graft(ssk, parent_sock); 3050 subflow->map_seq = READ_ONCE(msk->ack_seq); 3051 out: 3052 mptcp_event(MPTCP_EVENT_SUB_ESTABLISHED, msk, ssk, GFP_ATOMIC); 3053 return true; 3054 } 3055 3056 static void mptcp_shutdown(struct sock *sk, int how) 3057 { 3058 pr_debug("sk=%p, how=%d", sk, how); 3059 3060 if ((how & SEND_SHUTDOWN) && mptcp_close_state(sk)) 3061 __mptcp_wr_shutdown(sk); 3062 } 3063 3064 static struct proto mptcp_prot = { 3065 .name = "MPTCP", 3066 .owner = THIS_MODULE, 3067 .init = mptcp_init_sock, 3068 .disconnect = mptcp_disconnect, 3069 .close = mptcp_close, 3070 .accept = mptcp_accept, 3071 .setsockopt = mptcp_setsockopt, 3072 .getsockopt = mptcp_getsockopt, 3073 .shutdown = mptcp_shutdown, 3074 .destroy = mptcp_destroy, 3075 .sendmsg = mptcp_sendmsg, 3076 .recvmsg = mptcp_recvmsg, 3077 .release_cb = mptcp_release_cb, 3078 .hash = mptcp_hash, 3079 .unhash = mptcp_unhash, 3080 .get_port = mptcp_get_port, 3081 .sockets_allocated = &mptcp_sockets_allocated, 3082 .memory_allocated = &tcp_memory_allocated, 3083 .memory_pressure = &tcp_memory_pressure, 3084 .sysctl_wmem_offset = offsetof(struct net, ipv4.sysctl_tcp_wmem), 3085 .sysctl_rmem_offset = offsetof(struct net, ipv4.sysctl_tcp_rmem), 3086 .sysctl_mem = sysctl_tcp_mem, 3087 .obj_size = sizeof(struct mptcp_sock), 3088 .slab_flags = SLAB_TYPESAFE_BY_RCU, 3089 .no_autobind = true, 3090 }; 3091 3092 static int mptcp_bind(struct socket *sock, struct sockaddr *uaddr, int addr_len) 3093 { 3094 struct mptcp_sock *msk = mptcp_sk(sock->sk); 3095 struct socket *ssock; 3096 int err; 3097 3098 lock_sock(sock->sk); 3099 ssock = __mptcp_nmpc_socket(msk); 3100 if (!ssock) { 3101 err = -EINVAL; 3102 goto unlock; 3103 } 3104 3105 err = ssock->ops->bind(ssock, uaddr, addr_len); 3106 if (!err) 3107 mptcp_copy_inaddrs(sock->sk, ssock->sk); 3108 3109 unlock: 3110 release_sock(sock->sk); 3111 return err; 3112 } 3113 3114 static void mptcp_subflow_early_fallback(struct mptcp_sock *msk, 3115 struct mptcp_subflow_context *subflow) 3116 { 3117 subflow->request_mptcp = 0; 3118 __mptcp_do_fallback(msk); 3119 } 3120 3121 static int mptcp_stream_connect(struct socket *sock, struct sockaddr *uaddr, 3122 int addr_len, int flags) 3123 { 3124 struct mptcp_sock *msk = mptcp_sk(sock->sk); 3125 struct mptcp_subflow_context *subflow; 3126 struct socket *ssock; 3127 int err; 3128 3129 lock_sock(sock->sk); 3130 if (sock->state != SS_UNCONNECTED && msk->subflow) { 3131 /* pending connection or invalid state, let existing subflow 3132 * cope with that 3133 */ 3134 ssock = msk->subflow; 3135 goto do_connect; 3136 } 3137 3138 ssock = __mptcp_nmpc_socket(msk); 3139 if (!ssock) { 3140 err = -EINVAL; 3141 goto unlock; 3142 } 3143 3144 mptcp_token_destroy(msk); 3145 inet_sk_state_store(sock->sk, TCP_SYN_SENT); 3146 subflow = mptcp_subflow_ctx(ssock->sk); 3147 #ifdef CONFIG_TCP_MD5SIG 3148 /* no MPTCP if MD5SIG is enabled on this socket or we may run out of 3149 * TCP option space. 3150 */ 3151 if (rcu_access_pointer(tcp_sk(ssock->sk)->md5sig_info)) 3152 mptcp_subflow_early_fallback(msk, subflow); 3153 #endif 3154 if (subflow->request_mptcp && mptcp_token_new_connect(ssock->sk)) { 3155 MPTCP_INC_STATS(sock_net(ssock->sk), MPTCP_MIB_TOKENFALLBACKINIT); 3156 mptcp_subflow_early_fallback(msk, subflow); 3157 } 3158 if (likely(!__mptcp_check_fallback(msk))) 3159 MPTCP_INC_STATS(sock_net(sock->sk), MPTCP_MIB_MPCAPABLEACTIVE); 3160 3161 do_connect: 3162 err = ssock->ops->connect(ssock, uaddr, addr_len, flags); 3163 sock->state = ssock->state; 3164 3165 /* on successful connect, the msk state will be moved to established by 3166 * subflow_finish_connect() 3167 */ 3168 if (!err || err == -EINPROGRESS) 3169 mptcp_copy_inaddrs(sock->sk, ssock->sk); 3170 else 3171 inet_sk_state_store(sock->sk, inet_sk_state_load(ssock->sk)); 3172 3173 unlock: 3174 release_sock(sock->sk); 3175 return err; 3176 } 3177 3178 static int mptcp_listen(struct socket *sock, int backlog) 3179 { 3180 struct mptcp_sock *msk = mptcp_sk(sock->sk); 3181 struct socket *ssock; 3182 int err; 3183 3184 pr_debug("msk=%p", msk); 3185 3186 lock_sock(sock->sk); 3187 ssock = __mptcp_nmpc_socket(msk); 3188 if (!ssock) { 3189 err = -EINVAL; 3190 goto unlock; 3191 } 3192 3193 mptcp_token_destroy(msk); 3194 inet_sk_state_store(sock->sk, TCP_LISTEN); 3195 sock_set_flag(sock->sk, SOCK_RCU_FREE); 3196 3197 err = ssock->ops->listen(ssock, backlog); 3198 inet_sk_state_store(sock->sk, inet_sk_state_load(ssock->sk)); 3199 if (!err) 3200 mptcp_copy_inaddrs(sock->sk, ssock->sk); 3201 3202 unlock: 3203 release_sock(sock->sk); 3204 return err; 3205 } 3206 3207 static int mptcp_stream_accept(struct socket *sock, struct socket *newsock, 3208 int flags, bool kern) 3209 { 3210 struct mptcp_sock *msk = mptcp_sk(sock->sk); 3211 struct socket *ssock; 3212 int err; 3213 3214 pr_debug("msk=%p", msk); 3215 3216 lock_sock(sock->sk); 3217 if (sock->sk->sk_state != TCP_LISTEN) 3218 goto unlock_fail; 3219 3220 ssock = __mptcp_nmpc_socket(msk); 3221 if (!ssock) 3222 goto unlock_fail; 3223 3224 clear_bit(MPTCP_DATA_READY, &msk->flags); 3225 sock_hold(ssock->sk); 3226 release_sock(sock->sk); 3227 3228 err = ssock->ops->accept(sock, newsock, flags, kern); 3229 if (err == 0 && !mptcp_is_tcpsk(newsock->sk)) { 3230 struct mptcp_sock *msk = mptcp_sk(newsock->sk); 3231 struct mptcp_subflow_context *subflow; 3232 struct sock *newsk = newsock->sk; 3233 3234 lock_sock(newsk); 3235 3236 /* PM/worker can now acquire the first subflow socket 3237 * lock without racing with listener queue cleanup, 3238 * we can notify it, if needed. 3239 * 3240 * Even if remote has reset the initial subflow by now 3241 * the refcnt is still at least one. 3242 */ 3243 subflow = mptcp_subflow_ctx(msk->first); 3244 list_add(&subflow->node, &msk->conn_list); 3245 sock_hold(msk->first); 3246 if (mptcp_is_fully_established(newsk)) 3247 mptcp_pm_fully_established(msk, msk->first, GFP_KERNEL); 3248 3249 mptcp_copy_inaddrs(newsk, msk->first); 3250 mptcp_rcv_space_init(msk, msk->first); 3251 mptcp_propagate_sndbuf(newsk, msk->first); 3252 3253 /* set ssk->sk_socket of accept()ed flows to mptcp socket. 3254 * This is needed so NOSPACE flag can be set from tcp stack. 3255 */ 3256 mptcp_flush_join_list(msk); 3257 mptcp_for_each_subflow(msk, subflow) { 3258 struct sock *ssk = mptcp_subflow_tcp_sock(subflow); 3259 3260 if (!ssk->sk_socket) 3261 mptcp_sock_graft(ssk, newsock); 3262 } 3263 release_sock(newsk); 3264 } 3265 3266 if (inet_csk_listen_poll(ssock->sk)) 3267 set_bit(MPTCP_DATA_READY, &msk->flags); 3268 sock_put(ssock->sk); 3269 return err; 3270 3271 unlock_fail: 3272 release_sock(sock->sk); 3273 return -EINVAL; 3274 } 3275 3276 static __poll_t mptcp_check_readable(struct mptcp_sock *msk) 3277 { 3278 return test_bit(MPTCP_DATA_READY, &msk->flags) ? EPOLLIN | EPOLLRDNORM : 3279 0; 3280 } 3281 3282 static __poll_t mptcp_check_writeable(struct mptcp_sock *msk) 3283 { 3284 struct sock *sk = (struct sock *)msk; 3285 3286 if (unlikely(sk->sk_shutdown & SEND_SHUTDOWN)) 3287 return EPOLLOUT | EPOLLWRNORM; 3288 3289 if (sk_stream_is_writeable(sk)) 3290 return EPOLLOUT | EPOLLWRNORM; 3291 3292 mptcp_set_nospace(sk); 3293 smp_mb__after_atomic(); /* msk->flags is changed by write_space cb */ 3294 if (sk_stream_is_writeable(sk)) 3295 return EPOLLOUT | EPOLLWRNORM; 3296 3297 return 0; 3298 } 3299 3300 static __poll_t mptcp_poll(struct file *file, struct socket *sock, 3301 struct poll_table_struct *wait) 3302 { 3303 struct sock *sk = sock->sk; 3304 struct mptcp_sock *msk; 3305 __poll_t mask = 0; 3306 int state; 3307 3308 msk = mptcp_sk(sk); 3309 sock_poll_wait(file, sock, wait); 3310 3311 state = inet_sk_state_load(sk); 3312 pr_debug("msk=%p state=%d flags=%lx", msk, state, msk->flags); 3313 if (state == TCP_LISTEN) 3314 return mptcp_check_readable(msk); 3315 3316 if (state != TCP_SYN_SENT && state != TCP_SYN_RECV) { 3317 mask |= mptcp_check_readable(msk); 3318 mask |= mptcp_check_writeable(msk); 3319 } 3320 if (sk->sk_shutdown == SHUTDOWN_MASK || state == TCP_CLOSE) 3321 mask |= EPOLLHUP; 3322 if (sk->sk_shutdown & RCV_SHUTDOWN) 3323 mask |= EPOLLIN | EPOLLRDNORM | EPOLLRDHUP; 3324 3325 /* This barrier is coupled with smp_wmb() in tcp_reset() */ 3326 smp_rmb(); 3327 if (sk->sk_err) 3328 mask |= EPOLLERR; 3329 3330 return mask; 3331 } 3332 3333 static const struct proto_ops mptcp_stream_ops = { 3334 .family = PF_INET, 3335 .owner = THIS_MODULE, 3336 .release = inet_release, 3337 .bind = mptcp_bind, 3338 .connect = mptcp_stream_connect, 3339 .socketpair = sock_no_socketpair, 3340 .accept = mptcp_stream_accept, 3341 .getname = inet_getname, 3342 .poll = mptcp_poll, 3343 .ioctl = inet_ioctl, 3344 .gettstamp = sock_gettstamp, 3345 .listen = mptcp_listen, 3346 .shutdown = inet_shutdown, 3347 .setsockopt = sock_common_setsockopt, 3348 .getsockopt = sock_common_getsockopt, 3349 .sendmsg = inet_sendmsg, 3350 .recvmsg = inet_recvmsg, 3351 .mmap = sock_no_mmap, 3352 .sendpage = inet_sendpage, 3353 }; 3354 3355 static struct inet_protosw mptcp_protosw = { 3356 .type = SOCK_STREAM, 3357 .protocol = IPPROTO_MPTCP, 3358 .prot = &mptcp_prot, 3359 .ops = &mptcp_stream_ops, 3360 .flags = INET_PROTOSW_ICSK, 3361 }; 3362 3363 static int mptcp_napi_poll(struct napi_struct *napi, int budget) 3364 { 3365 struct mptcp_delegated_action *delegated; 3366 struct mptcp_subflow_context *subflow; 3367 int work_done = 0; 3368 3369 delegated = container_of(napi, struct mptcp_delegated_action, napi); 3370 while ((subflow = mptcp_subflow_delegated_next(delegated)) != NULL) { 3371 struct sock *ssk = mptcp_subflow_tcp_sock(subflow); 3372 3373 bh_lock_sock_nested(ssk); 3374 if (!sock_owned_by_user(ssk) && 3375 mptcp_subflow_has_delegated_action(subflow)) 3376 mptcp_subflow_process_delegated(ssk); 3377 /* ... elsewhere tcp_release_cb_override already processed 3378 * the action or will do at next release_sock(). 3379 * In both case must dequeue the subflow here - on the same 3380 * CPU that scheduled it. 3381 */ 3382 bh_unlock_sock(ssk); 3383 sock_put(ssk); 3384 3385 if (++work_done == budget) 3386 return budget; 3387 } 3388 3389 /* always provide a 0 'work_done' argument, so that napi_complete_done 3390 * will not try accessing the NULL napi->dev ptr 3391 */ 3392 napi_complete_done(napi, 0); 3393 return work_done; 3394 } 3395 3396 void __init mptcp_proto_init(void) 3397 { 3398 struct mptcp_delegated_action *delegated; 3399 int cpu; 3400 3401 mptcp_prot.h.hashinfo = tcp_prot.h.hashinfo; 3402 3403 if (percpu_counter_init(&mptcp_sockets_allocated, 0, GFP_KERNEL)) 3404 panic("Failed to allocate MPTCP pcpu counter\n"); 3405 3406 init_dummy_netdev(&mptcp_napi_dev); 3407 for_each_possible_cpu(cpu) { 3408 delegated = per_cpu_ptr(&mptcp_delegated_actions, cpu); 3409 INIT_LIST_HEAD(&delegated->head); 3410 netif_tx_napi_add(&mptcp_napi_dev, &delegated->napi, mptcp_napi_poll, 3411 NAPI_POLL_WEIGHT); 3412 napi_enable(&delegated->napi); 3413 } 3414 3415 mptcp_subflow_init(); 3416 mptcp_pm_init(); 3417 mptcp_token_init(); 3418 3419 if (proto_register(&mptcp_prot, 1) != 0) 3420 panic("Failed to register MPTCP proto.\n"); 3421 3422 inet_register_protosw(&mptcp_protosw); 3423 3424 BUILD_BUG_ON(sizeof(struct mptcp_skb_cb) > sizeof_field(struct sk_buff, cb)); 3425 } 3426 3427 #if IS_ENABLED(CONFIG_MPTCP_IPV6) 3428 static const struct proto_ops mptcp_v6_stream_ops = { 3429 .family = PF_INET6, 3430 .owner = THIS_MODULE, 3431 .release = inet6_release, 3432 .bind = mptcp_bind, 3433 .connect = mptcp_stream_connect, 3434 .socketpair = sock_no_socketpair, 3435 .accept = mptcp_stream_accept, 3436 .getname = inet6_getname, 3437 .poll = mptcp_poll, 3438 .ioctl = inet6_ioctl, 3439 .gettstamp = sock_gettstamp, 3440 .listen = mptcp_listen, 3441 .shutdown = inet_shutdown, 3442 .setsockopt = sock_common_setsockopt, 3443 .getsockopt = sock_common_getsockopt, 3444 .sendmsg = inet6_sendmsg, 3445 .recvmsg = inet6_recvmsg, 3446 .mmap = sock_no_mmap, 3447 .sendpage = inet_sendpage, 3448 #ifdef CONFIG_COMPAT 3449 .compat_ioctl = inet6_compat_ioctl, 3450 #endif 3451 }; 3452 3453 static struct proto mptcp_v6_prot; 3454 3455 static void mptcp_v6_destroy(struct sock *sk) 3456 { 3457 mptcp_destroy(sk); 3458 inet6_destroy_sock(sk); 3459 } 3460 3461 static struct inet_protosw mptcp_v6_protosw = { 3462 .type = SOCK_STREAM, 3463 .protocol = IPPROTO_MPTCP, 3464 .prot = &mptcp_v6_prot, 3465 .ops = &mptcp_v6_stream_ops, 3466 .flags = INET_PROTOSW_ICSK, 3467 }; 3468 3469 int __init mptcp_proto_v6_init(void) 3470 { 3471 int err; 3472 3473 mptcp_v6_prot = mptcp_prot; 3474 strcpy(mptcp_v6_prot.name, "MPTCPv6"); 3475 mptcp_v6_prot.slab = NULL; 3476 mptcp_v6_prot.destroy = mptcp_v6_destroy; 3477 mptcp_v6_prot.obj_size = sizeof(struct mptcp6_sock); 3478 3479 err = proto_register(&mptcp_v6_prot, 1); 3480 if (err) 3481 return err; 3482 3483 err = inet6_register_protosw(&mptcp_v6_protosw); 3484 if (err) 3485 proto_unregister(&mptcp_v6_prot); 3486 3487 return err; 3488 } 3489 #endif 3490