1 // SPDX-License-Identifier: GPL-2.0 2 /* Multipath TCP 3 * 4 * Copyright (c) 2017 - 2019, Intel Corporation. 5 */ 6 7 #define pr_fmt(fmt) "MPTCP: " fmt 8 9 #include <linux/kernel.h> 10 #include <linux/module.h> 11 #include <linux/netdevice.h> 12 #include <crypto/algapi.h> 13 #include <crypto/sha2.h> 14 #include <net/sock.h> 15 #include <net/inet_common.h> 16 #include <net/inet_hashtables.h> 17 #include <net/protocol.h> 18 #include <net/tcp.h> 19 #if IS_ENABLED(CONFIG_MPTCP_IPV6) 20 #include <net/ip6_route.h> 21 #include <net/transp_v6.h> 22 #endif 23 #include <net/mptcp.h> 24 #include <uapi/linux/mptcp.h> 25 #include "protocol.h" 26 #include "mib.h" 27 28 static void mptcp_subflow_ops_undo_override(struct sock *ssk); 29 30 static void SUBFLOW_REQ_INC_STATS(struct request_sock *req, 31 enum linux_mptcp_mib_field field) 32 { 33 MPTCP_INC_STATS(sock_net(req_to_sk(req)), field); 34 } 35 36 static void subflow_req_destructor(struct request_sock *req) 37 { 38 struct mptcp_subflow_request_sock *subflow_req = mptcp_subflow_rsk(req); 39 40 pr_debug("subflow_req=%p", subflow_req); 41 42 if (subflow_req->msk) 43 sock_put((struct sock *)subflow_req->msk); 44 45 mptcp_token_destroy_request(req); 46 tcp_request_sock_ops.destructor(req); 47 } 48 49 static void subflow_generate_hmac(u64 key1, u64 key2, u32 nonce1, u32 nonce2, 50 void *hmac) 51 { 52 u8 msg[8]; 53 54 put_unaligned_be32(nonce1, &msg[0]); 55 put_unaligned_be32(nonce2, &msg[4]); 56 57 mptcp_crypto_hmac_sha(key1, key2, msg, 8, hmac); 58 } 59 60 static bool mptcp_can_accept_new_subflow(const struct mptcp_sock *msk) 61 { 62 return mptcp_is_fully_established((void *)msk) && 63 READ_ONCE(msk->pm.accept_subflow); 64 } 65 66 /* validate received token and create truncated hmac and nonce for SYN-ACK */ 67 static void subflow_req_create_thmac(struct mptcp_subflow_request_sock *subflow_req) 68 { 69 struct mptcp_sock *msk = subflow_req->msk; 70 u8 hmac[SHA256_DIGEST_SIZE]; 71 72 get_random_bytes(&subflow_req->local_nonce, sizeof(u32)); 73 74 subflow_generate_hmac(msk->local_key, msk->remote_key, 75 subflow_req->local_nonce, 76 subflow_req->remote_nonce, hmac); 77 78 subflow_req->thmac = get_unaligned_be64(hmac); 79 } 80 81 static struct mptcp_sock *subflow_token_join_request(struct request_sock *req) 82 { 83 struct mptcp_subflow_request_sock *subflow_req = mptcp_subflow_rsk(req); 84 struct mptcp_sock *msk; 85 int local_id; 86 87 msk = mptcp_token_get_sock(subflow_req->token); 88 if (!msk) { 89 SUBFLOW_REQ_INC_STATS(req, MPTCP_MIB_JOINNOTOKEN); 90 return NULL; 91 } 92 93 local_id = mptcp_pm_get_local_id(msk, (struct sock_common *)req); 94 if (local_id < 0) { 95 sock_put((struct sock *)msk); 96 return NULL; 97 } 98 subflow_req->local_id = local_id; 99 100 return msk; 101 } 102 103 static void subflow_init_req(struct request_sock *req, const struct sock *sk_listener) 104 { 105 struct mptcp_subflow_request_sock *subflow_req = mptcp_subflow_rsk(req); 106 107 subflow_req->mp_capable = 0; 108 subflow_req->mp_join = 0; 109 subflow_req->msk = NULL; 110 mptcp_token_init_request(req); 111 } 112 113 static bool subflow_use_different_sport(struct mptcp_sock *msk, const struct sock *sk) 114 { 115 return inet_sk(sk)->inet_sport != inet_sk((struct sock *)msk)->inet_sport; 116 } 117 118 /* Init mptcp request socket. 119 * 120 * Returns an error code if a JOIN has failed and a TCP reset 121 * should be sent. 122 */ 123 static int subflow_check_req(struct request_sock *req, 124 const struct sock *sk_listener, 125 struct sk_buff *skb) 126 { 127 struct mptcp_subflow_context *listener = mptcp_subflow_ctx(sk_listener); 128 struct mptcp_subflow_request_sock *subflow_req = mptcp_subflow_rsk(req); 129 struct mptcp_options_received mp_opt; 130 131 pr_debug("subflow_req=%p, listener=%p", subflow_req, listener); 132 133 #ifdef CONFIG_TCP_MD5SIG 134 /* no MPTCP if MD5SIG is enabled on this socket or we may run out of 135 * TCP option space. 136 */ 137 if (rcu_access_pointer(tcp_sk(sk_listener)->md5sig_info)) 138 return -EINVAL; 139 #endif 140 141 mptcp_get_options(skb, &mp_opt); 142 143 if (mp_opt.mp_capable) { 144 SUBFLOW_REQ_INC_STATS(req, MPTCP_MIB_MPCAPABLEPASSIVE); 145 146 if (mp_opt.mp_join) 147 return 0; 148 } else if (mp_opt.mp_join) { 149 SUBFLOW_REQ_INC_STATS(req, MPTCP_MIB_JOINSYNRX); 150 } 151 152 if (mp_opt.mp_capable && listener->request_mptcp) { 153 int err, retries = 4; 154 155 subflow_req->ssn_offset = TCP_SKB_CB(skb)->seq; 156 again: 157 do { 158 get_random_bytes(&subflow_req->local_key, sizeof(subflow_req->local_key)); 159 } while (subflow_req->local_key == 0); 160 161 if (unlikely(req->syncookie)) { 162 mptcp_crypto_key_sha(subflow_req->local_key, 163 &subflow_req->token, 164 &subflow_req->idsn); 165 if (mptcp_token_exists(subflow_req->token)) { 166 if (retries-- > 0) 167 goto again; 168 } else { 169 subflow_req->mp_capable = 1; 170 } 171 return 0; 172 } 173 174 err = mptcp_token_new_request(req); 175 if (err == 0) 176 subflow_req->mp_capable = 1; 177 else if (retries-- > 0) 178 goto again; 179 180 } else if (mp_opt.mp_join && listener->request_mptcp) { 181 subflow_req->ssn_offset = TCP_SKB_CB(skb)->seq; 182 subflow_req->mp_join = 1; 183 subflow_req->backup = mp_opt.backup; 184 subflow_req->remote_id = mp_opt.join_id; 185 subflow_req->token = mp_opt.token; 186 subflow_req->remote_nonce = mp_opt.nonce; 187 subflow_req->msk = subflow_token_join_request(req); 188 189 /* Can't fall back to TCP in this case. */ 190 if (!subflow_req->msk) 191 return -EPERM; 192 193 if (subflow_use_different_sport(subflow_req->msk, sk_listener)) { 194 pr_debug("syn inet_sport=%d %d", 195 ntohs(inet_sk(sk_listener)->inet_sport), 196 ntohs(inet_sk((struct sock *)subflow_req->msk)->inet_sport)); 197 if (!mptcp_pm_sport_in_anno_list(subflow_req->msk, sk_listener)) { 198 sock_put((struct sock *)subflow_req->msk); 199 mptcp_token_destroy_request(req); 200 tcp_request_sock_ops.destructor(req); 201 subflow_req->msk = NULL; 202 subflow_req->mp_join = 0; 203 SUBFLOW_REQ_INC_STATS(req, MPTCP_MIB_MISMATCHPORTSYNRX); 204 return -EPERM; 205 } 206 SUBFLOW_REQ_INC_STATS(req, MPTCP_MIB_JOINPORTSYNRX); 207 } 208 209 subflow_req_create_thmac(subflow_req); 210 211 if (unlikely(req->syncookie)) { 212 if (mptcp_can_accept_new_subflow(subflow_req->msk)) 213 subflow_init_req_cookie_join_save(subflow_req, skb); 214 } 215 216 pr_debug("token=%u, remote_nonce=%u msk=%p", subflow_req->token, 217 subflow_req->remote_nonce, subflow_req->msk); 218 } 219 220 return 0; 221 } 222 223 int mptcp_subflow_init_cookie_req(struct request_sock *req, 224 const struct sock *sk_listener, 225 struct sk_buff *skb) 226 { 227 struct mptcp_subflow_context *listener = mptcp_subflow_ctx(sk_listener); 228 struct mptcp_subflow_request_sock *subflow_req = mptcp_subflow_rsk(req); 229 struct mptcp_options_received mp_opt; 230 int err; 231 232 subflow_init_req(req, sk_listener); 233 mptcp_get_options(skb, &mp_opt); 234 235 if (mp_opt.mp_capable && mp_opt.mp_join) 236 return -EINVAL; 237 238 if (mp_opt.mp_capable && listener->request_mptcp) { 239 if (mp_opt.sndr_key == 0) 240 return -EINVAL; 241 242 subflow_req->local_key = mp_opt.rcvr_key; 243 err = mptcp_token_new_request(req); 244 if (err) 245 return err; 246 247 subflow_req->mp_capable = 1; 248 subflow_req->ssn_offset = TCP_SKB_CB(skb)->seq - 1; 249 } else if (mp_opt.mp_join && listener->request_mptcp) { 250 if (!mptcp_token_join_cookie_init_state(subflow_req, skb)) 251 return -EINVAL; 252 253 if (mptcp_can_accept_new_subflow(subflow_req->msk)) 254 subflow_req->mp_join = 1; 255 256 subflow_req->ssn_offset = TCP_SKB_CB(skb)->seq - 1; 257 } 258 259 return 0; 260 } 261 EXPORT_SYMBOL_GPL(mptcp_subflow_init_cookie_req); 262 263 static struct dst_entry *subflow_v4_route_req(const struct sock *sk, 264 struct sk_buff *skb, 265 struct flowi *fl, 266 struct request_sock *req) 267 { 268 struct dst_entry *dst; 269 int err; 270 271 tcp_rsk(req)->is_mptcp = 1; 272 subflow_init_req(req, sk); 273 274 dst = tcp_request_sock_ipv4_ops.route_req(sk, skb, fl, req); 275 if (!dst) 276 return NULL; 277 278 err = subflow_check_req(req, sk, skb); 279 if (err == 0) 280 return dst; 281 282 dst_release(dst); 283 if (!req->syncookie) 284 tcp_request_sock_ops.send_reset(sk, skb); 285 return NULL; 286 } 287 288 #if IS_ENABLED(CONFIG_MPTCP_IPV6) 289 static struct dst_entry *subflow_v6_route_req(const struct sock *sk, 290 struct sk_buff *skb, 291 struct flowi *fl, 292 struct request_sock *req) 293 { 294 struct dst_entry *dst; 295 int err; 296 297 tcp_rsk(req)->is_mptcp = 1; 298 subflow_init_req(req, sk); 299 300 dst = tcp_request_sock_ipv6_ops.route_req(sk, skb, fl, req); 301 if (!dst) 302 return NULL; 303 304 err = subflow_check_req(req, sk, skb); 305 if (err == 0) 306 return dst; 307 308 dst_release(dst); 309 if (!req->syncookie) 310 tcp6_request_sock_ops.send_reset(sk, skb); 311 return NULL; 312 } 313 #endif 314 315 /* validate received truncated hmac and create hmac for third ACK */ 316 static bool subflow_thmac_valid(struct mptcp_subflow_context *subflow) 317 { 318 u8 hmac[SHA256_DIGEST_SIZE]; 319 u64 thmac; 320 321 subflow_generate_hmac(subflow->remote_key, subflow->local_key, 322 subflow->remote_nonce, subflow->local_nonce, 323 hmac); 324 325 thmac = get_unaligned_be64(hmac); 326 pr_debug("subflow=%p, token=%u, thmac=%llu, subflow->thmac=%llu\n", 327 subflow, subflow->token, 328 (unsigned long long)thmac, 329 (unsigned long long)subflow->thmac); 330 331 return thmac == subflow->thmac; 332 } 333 334 void mptcp_subflow_reset(struct sock *ssk) 335 { 336 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk); 337 struct sock *sk = subflow->conn; 338 339 /* must hold: tcp_done() could drop last reference on parent */ 340 sock_hold(sk); 341 342 tcp_set_state(ssk, TCP_CLOSE); 343 tcp_send_active_reset(ssk, GFP_ATOMIC); 344 tcp_done(ssk); 345 if (!test_and_set_bit(MPTCP_WORK_CLOSE_SUBFLOW, &mptcp_sk(sk)->flags) && 346 schedule_work(&mptcp_sk(sk)->work)) 347 return; /* worker will put sk for us */ 348 349 sock_put(sk); 350 } 351 352 static bool subflow_use_different_dport(struct mptcp_sock *msk, const struct sock *sk) 353 { 354 return inet_sk(sk)->inet_dport != inet_sk((struct sock *)msk)->inet_dport; 355 } 356 357 static void subflow_finish_connect(struct sock *sk, const struct sk_buff *skb) 358 { 359 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(sk); 360 struct mptcp_options_received mp_opt; 361 struct sock *parent = subflow->conn; 362 363 subflow->icsk_af_ops->sk_rx_dst_set(sk, skb); 364 365 if (inet_sk_state_load(parent) == TCP_SYN_SENT) { 366 inet_sk_state_store(parent, TCP_ESTABLISHED); 367 parent->sk_state_change(parent); 368 } 369 370 /* be sure no special action on any packet other than syn-ack */ 371 if (subflow->conn_finished) 372 return; 373 374 mptcp_propagate_sndbuf(parent, sk); 375 subflow->rel_write_seq = 1; 376 subflow->conn_finished = 1; 377 subflow->ssn_offset = TCP_SKB_CB(skb)->seq; 378 pr_debug("subflow=%p synack seq=%x", subflow, subflow->ssn_offset); 379 380 mptcp_get_options(skb, &mp_opt); 381 if (subflow->request_mptcp) { 382 if (!mp_opt.mp_capable) { 383 MPTCP_INC_STATS(sock_net(sk), 384 MPTCP_MIB_MPCAPABLEACTIVEFALLBACK); 385 mptcp_do_fallback(sk); 386 pr_fallback(mptcp_sk(subflow->conn)); 387 goto fallback; 388 } 389 390 subflow->mp_capable = 1; 391 subflow->can_ack = 1; 392 subflow->remote_key = mp_opt.sndr_key; 393 pr_debug("subflow=%p, remote_key=%llu", subflow, 394 subflow->remote_key); 395 mptcp_finish_connect(sk); 396 } else if (subflow->request_join) { 397 u8 hmac[SHA256_DIGEST_SIZE]; 398 399 if (!mp_opt.mp_join) 400 goto do_reset; 401 402 subflow->thmac = mp_opt.thmac; 403 subflow->remote_nonce = mp_opt.nonce; 404 pr_debug("subflow=%p, thmac=%llu, remote_nonce=%u", subflow, 405 subflow->thmac, subflow->remote_nonce); 406 407 if (!subflow_thmac_valid(subflow)) { 408 MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_JOINACKMAC); 409 goto do_reset; 410 } 411 412 subflow_generate_hmac(subflow->local_key, subflow->remote_key, 413 subflow->local_nonce, 414 subflow->remote_nonce, 415 hmac); 416 memcpy(subflow->hmac, hmac, MPTCPOPT_HMAC_LEN); 417 418 if (!mptcp_finish_join(sk)) 419 goto do_reset; 420 421 subflow->mp_join = 1; 422 MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_JOINSYNACKRX); 423 424 if (subflow_use_different_dport(mptcp_sk(parent), sk)) { 425 pr_debug("synack inet_dport=%d %d", 426 ntohs(inet_sk(sk)->inet_dport), 427 ntohs(inet_sk(parent)->inet_dport)); 428 MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_JOINPORTSYNACKRX); 429 } 430 } else if (mptcp_check_fallback(sk)) { 431 fallback: 432 mptcp_rcv_space_init(mptcp_sk(parent), sk); 433 } 434 return; 435 436 do_reset: 437 mptcp_subflow_reset(sk); 438 } 439 440 struct request_sock_ops mptcp_subflow_request_sock_ops; 441 EXPORT_SYMBOL_GPL(mptcp_subflow_request_sock_ops); 442 static struct tcp_request_sock_ops subflow_request_sock_ipv4_ops; 443 444 static int subflow_v4_conn_request(struct sock *sk, struct sk_buff *skb) 445 { 446 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(sk); 447 448 pr_debug("subflow=%p", subflow); 449 450 /* Never answer to SYNs sent to broadcast or multicast */ 451 if (skb_rtable(skb)->rt_flags & (RTCF_BROADCAST | RTCF_MULTICAST)) 452 goto drop; 453 454 return tcp_conn_request(&mptcp_subflow_request_sock_ops, 455 &subflow_request_sock_ipv4_ops, 456 sk, skb); 457 drop: 458 tcp_listendrop(sk); 459 return 0; 460 } 461 462 #if IS_ENABLED(CONFIG_MPTCP_IPV6) 463 static struct tcp_request_sock_ops subflow_request_sock_ipv6_ops; 464 static struct inet_connection_sock_af_ops subflow_v6_specific; 465 static struct inet_connection_sock_af_ops subflow_v6m_specific; 466 static struct proto tcpv6_prot_override; 467 468 static int subflow_v6_conn_request(struct sock *sk, struct sk_buff *skb) 469 { 470 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(sk); 471 472 pr_debug("subflow=%p", subflow); 473 474 if (skb->protocol == htons(ETH_P_IP)) 475 return subflow_v4_conn_request(sk, skb); 476 477 if (!ipv6_unicast_destination(skb)) 478 goto drop; 479 480 if (ipv6_addr_v4mapped(&ipv6_hdr(skb)->saddr)) { 481 __IP6_INC_STATS(sock_net(sk), NULL, IPSTATS_MIB_INHDRERRORS); 482 return 0; 483 } 484 485 return tcp_conn_request(&mptcp_subflow_request_sock_ops, 486 &subflow_request_sock_ipv6_ops, sk, skb); 487 488 drop: 489 tcp_listendrop(sk); 490 return 0; /* don't send reset */ 491 } 492 #endif 493 494 /* validate hmac received in third ACK */ 495 static bool subflow_hmac_valid(const struct request_sock *req, 496 const struct mptcp_options_received *mp_opt) 497 { 498 const struct mptcp_subflow_request_sock *subflow_req; 499 u8 hmac[SHA256_DIGEST_SIZE]; 500 struct mptcp_sock *msk; 501 502 subflow_req = mptcp_subflow_rsk(req); 503 msk = subflow_req->msk; 504 if (!msk) 505 return false; 506 507 subflow_generate_hmac(msk->remote_key, msk->local_key, 508 subflow_req->remote_nonce, 509 subflow_req->local_nonce, hmac); 510 511 return !crypto_memneq(hmac, mp_opt->hmac, MPTCPOPT_HMAC_LEN); 512 } 513 514 static void mptcp_sock_destruct(struct sock *sk) 515 { 516 /* if new mptcp socket isn't accepted, it is free'd 517 * from the tcp listener sockets request queue, linked 518 * from req->sk. The tcp socket is released. 519 * This calls the ULP release function which will 520 * also remove the mptcp socket, via 521 * sock_put(ctx->conn). 522 * 523 * Problem is that the mptcp socket will be in 524 * ESTABLISHED state and will not have the SOCK_DEAD flag. 525 * Both result in warnings from inet_sock_destruct. 526 */ 527 528 if (sk->sk_state == TCP_ESTABLISHED) { 529 sk->sk_state = TCP_CLOSE; 530 WARN_ON_ONCE(sk->sk_socket); 531 sock_orphan(sk); 532 } 533 534 mptcp_destroy_common(mptcp_sk(sk)); 535 inet_sock_destruct(sk); 536 } 537 538 static void mptcp_force_close(struct sock *sk) 539 { 540 inet_sk_state_store(sk, TCP_CLOSE); 541 sk_common_release(sk); 542 } 543 544 static void subflow_ulp_fallback(struct sock *sk, 545 struct mptcp_subflow_context *old_ctx) 546 { 547 struct inet_connection_sock *icsk = inet_csk(sk); 548 549 mptcp_subflow_tcp_fallback(sk, old_ctx); 550 icsk->icsk_ulp_ops = NULL; 551 rcu_assign_pointer(icsk->icsk_ulp_data, NULL); 552 tcp_sk(sk)->is_mptcp = 0; 553 554 mptcp_subflow_ops_undo_override(sk); 555 } 556 557 static void subflow_drop_ctx(struct sock *ssk) 558 { 559 struct mptcp_subflow_context *ctx = mptcp_subflow_ctx(ssk); 560 561 if (!ctx) 562 return; 563 564 subflow_ulp_fallback(ssk, ctx); 565 if (ctx->conn) 566 sock_put(ctx->conn); 567 568 kfree_rcu(ctx, rcu); 569 } 570 571 void mptcp_subflow_fully_established(struct mptcp_subflow_context *subflow, 572 struct mptcp_options_received *mp_opt) 573 { 574 struct mptcp_sock *msk = mptcp_sk(subflow->conn); 575 576 subflow->remote_key = mp_opt->sndr_key; 577 subflow->fully_established = 1; 578 subflow->can_ack = 1; 579 WRITE_ONCE(msk->fully_established, true); 580 } 581 582 static struct sock *subflow_syn_recv_sock(const struct sock *sk, 583 struct sk_buff *skb, 584 struct request_sock *req, 585 struct dst_entry *dst, 586 struct request_sock *req_unhash, 587 bool *own_req) 588 { 589 struct mptcp_subflow_context *listener = mptcp_subflow_ctx(sk); 590 struct mptcp_subflow_request_sock *subflow_req; 591 struct mptcp_options_received mp_opt; 592 bool fallback, fallback_is_fatal; 593 struct sock *new_msk = NULL; 594 struct sock *child; 595 596 pr_debug("listener=%p, req=%p, conn=%p", listener, req, listener->conn); 597 598 /* After child creation we must look for 'mp_capable' even when options 599 * are not parsed 600 */ 601 mp_opt.mp_capable = 0; 602 603 /* hopefully temporary handling for MP_JOIN+syncookie */ 604 subflow_req = mptcp_subflow_rsk(req); 605 fallback_is_fatal = tcp_rsk(req)->is_mptcp && subflow_req->mp_join; 606 fallback = !tcp_rsk(req)->is_mptcp; 607 if (fallback) 608 goto create_child; 609 610 /* if the sk is MP_CAPABLE, we try to fetch the client key */ 611 if (subflow_req->mp_capable) { 612 if (TCP_SKB_CB(skb)->seq != subflow_req->ssn_offset + 1) { 613 /* here we can receive and accept an in-window, 614 * out-of-order pkt, which will not carry the MP_CAPABLE 615 * opt even on mptcp enabled paths 616 */ 617 goto create_msk; 618 } 619 620 mptcp_get_options(skb, &mp_opt); 621 if (!mp_opt.mp_capable) { 622 fallback = true; 623 goto create_child; 624 } 625 626 create_msk: 627 new_msk = mptcp_sk_clone(listener->conn, &mp_opt, req); 628 if (!new_msk) 629 fallback = true; 630 } else if (subflow_req->mp_join) { 631 mptcp_get_options(skb, &mp_opt); 632 if (!mp_opt.mp_join || !subflow_hmac_valid(req, &mp_opt) || 633 !mptcp_can_accept_new_subflow(subflow_req->msk)) { 634 SUBFLOW_REQ_INC_STATS(req, MPTCP_MIB_JOINACKMAC); 635 fallback = true; 636 } 637 } 638 639 create_child: 640 child = listener->icsk_af_ops->syn_recv_sock(sk, skb, req, dst, 641 req_unhash, own_req); 642 643 if (child && *own_req) { 644 struct mptcp_subflow_context *ctx = mptcp_subflow_ctx(child); 645 646 tcp_rsk(req)->drop_req = false; 647 648 /* we need to fallback on ctx allocation failure and on pre-reqs 649 * checking above. In the latter scenario we additionally need 650 * to reset the context to non MPTCP status. 651 */ 652 if (!ctx || fallback) { 653 if (fallback_is_fatal) 654 goto dispose_child; 655 656 subflow_drop_ctx(child); 657 goto out; 658 } 659 660 if (ctx->mp_capable) { 661 /* this can't race with mptcp_close(), as the msk is 662 * not yet exposted to user-space 663 */ 664 inet_sk_state_store((void *)new_msk, TCP_ESTABLISHED); 665 666 /* record the newly created socket as the first msk 667 * subflow, but don't link it yet into conn_list 668 */ 669 WRITE_ONCE(mptcp_sk(new_msk)->first, child); 670 671 /* new mpc subflow takes ownership of the newly 672 * created mptcp socket 673 */ 674 new_msk->sk_destruct = mptcp_sock_destruct; 675 mptcp_pm_new_connection(mptcp_sk(new_msk), child, 1); 676 mptcp_token_accept(subflow_req, mptcp_sk(new_msk)); 677 ctx->conn = new_msk; 678 new_msk = NULL; 679 680 /* with OoO packets we can reach here without ingress 681 * mpc option 682 */ 683 if (mp_opt.mp_capable) 684 mptcp_subflow_fully_established(ctx, &mp_opt); 685 } else if (ctx->mp_join) { 686 struct mptcp_sock *owner; 687 688 owner = subflow_req->msk; 689 if (!owner) 690 goto dispose_child; 691 692 /* move the msk reference ownership to the subflow */ 693 subflow_req->msk = NULL; 694 ctx->conn = (struct sock *)owner; 695 696 if (subflow_use_different_sport(owner, sk)) { 697 pr_debug("ack inet_sport=%d %d", 698 ntohs(inet_sk(sk)->inet_sport), 699 ntohs(inet_sk((struct sock *)owner)->inet_sport)); 700 if (!mptcp_pm_sport_in_anno_list(owner, sk)) { 701 SUBFLOW_REQ_INC_STATS(req, MPTCP_MIB_MISMATCHPORTACKRX); 702 goto dispose_child; 703 } 704 SUBFLOW_REQ_INC_STATS(req, MPTCP_MIB_JOINPORTACKRX); 705 } 706 707 if (!mptcp_finish_join(child)) 708 goto dispose_child; 709 710 SUBFLOW_REQ_INC_STATS(req, MPTCP_MIB_JOINACKRX); 711 tcp_rsk(req)->drop_req = true; 712 } 713 } 714 715 out: 716 /* dispose of the left over mptcp master, if any */ 717 if (unlikely(new_msk)) 718 mptcp_force_close(new_msk); 719 720 /* check for expected invariant - should never trigger, just help 721 * catching eariler subtle bugs 722 */ 723 WARN_ON_ONCE(child && *own_req && tcp_sk(child)->is_mptcp && 724 (!mptcp_subflow_ctx(child) || 725 !mptcp_subflow_ctx(child)->conn)); 726 return child; 727 728 dispose_child: 729 subflow_drop_ctx(child); 730 tcp_rsk(req)->drop_req = true; 731 inet_csk_prepare_for_destroy_sock(child); 732 tcp_done(child); 733 req->rsk_ops->send_reset(sk, skb); 734 735 /* The last child reference will be released by the caller */ 736 return child; 737 } 738 739 static struct inet_connection_sock_af_ops subflow_specific; 740 static struct proto tcp_prot_override; 741 742 enum mapping_status { 743 MAPPING_OK, 744 MAPPING_INVALID, 745 MAPPING_EMPTY, 746 MAPPING_DATA_FIN, 747 MAPPING_DUMMY 748 }; 749 750 static u64 expand_seq(u64 old_seq, u16 old_data_len, u64 seq) 751 { 752 if ((u32)seq == (u32)old_seq) 753 return old_seq; 754 755 /* Assume map covers data not mapped yet. */ 756 return seq | ((old_seq + old_data_len + 1) & GENMASK_ULL(63, 32)); 757 } 758 759 static void warn_bad_map(struct mptcp_subflow_context *subflow, u32 ssn) 760 { 761 WARN_ONCE(1, "Bad mapping: ssn=%d map_seq=%d map_data_len=%d", 762 ssn, subflow->map_subflow_seq, subflow->map_data_len); 763 } 764 765 static bool skb_is_fully_mapped(struct sock *ssk, struct sk_buff *skb) 766 { 767 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk); 768 unsigned int skb_consumed; 769 770 skb_consumed = tcp_sk(ssk)->copied_seq - TCP_SKB_CB(skb)->seq; 771 if (WARN_ON_ONCE(skb_consumed >= skb->len)) 772 return true; 773 774 return skb->len - skb_consumed <= subflow->map_data_len - 775 mptcp_subflow_get_map_offset(subflow); 776 } 777 778 static bool validate_mapping(struct sock *ssk, struct sk_buff *skb) 779 { 780 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk); 781 u32 ssn = tcp_sk(ssk)->copied_seq - subflow->ssn_offset; 782 783 if (unlikely(before(ssn, subflow->map_subflow_seq))) { 784 /* Mapping covers data later in the subflow stream, 785 * currently unsupported. 786 */ 787 warn_bad_map(subflow, ssn); 788 return false; 789 } 790 if (unlikely(!before(ssn, subflow->map_subflow_seq + 791 subflow->map_data_len))) { 792 /* Mapping does covers past subflow data, invalid */ 793 warn_bad_map(subflow, ssn + skb->len); 794 return false; 795 } 796 return true; 797 } 798 799 static enum mapping_status get_mapping_status(struct sock *ssk, 800 struct mptcp_sock *msk) 801 { 802 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk); 803 struct mptcp_ext *mpext; 804 struct sk_buff *skb; 805 u16 data_len; 806 u64 map_seq; 807 808 skb = skb_peek(&ssk->sk_receive_queue); 809 if (!skb) 810 return MAPPING_EMPTY; 811 812 if (mptcp_check_fallback(ssk)) 813 return MAPPING_DUMMY; 814 815 mpext = mptcp_get_ext(skb); 816 if (!mpext || !mpext->use_map) { 817 if (!subflow->map_valid && !skb->len) { 818 /* the TCP stack deliver 0 len FIN pkt to the receive 819 * queue, that is the only 0len pkts ever expected here, 820 * and we can admit no mapping only for 0 len pkts 821 */ 822 if (!(TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN)) 823 WARN_ONCE(1, "0len seq %d:%d flags %x", 824 TCP_SKB_CB(skb)->seq, 825 TCP_SKB_CB(skb)->end_seq, 826 TCP_SKB_CB(skb)->tcp_flags); 827 sk_eat_skb(ssk, skb); 828 return MAPPING_EMPTY; 829 } 830 831 if (!subflow->map_valid) 832 return MAPPING_INVALID; 833 834 goto validate_seq; 835 } 836 837 pr_debug("seq=%llu is64=%d ssn=%u data_len=%u data_fin=%d", 838 mpext->data_seq, mpext->dsn64, mpext->subflow_seq, 839 mpext->data_len, mpext->data_fin); 840 841 data_len = mpext->data_len; 842 if (data_len == 0) { 843 pr_err("Infinite mapping not handled"); 844 MPTCP_INC_STATS(sock_net(ssk), MPTCP_MIB_INFINITEMAPRX); 845 return MAPPING_INVALID; 846 } 847 848 if (mpext->data_fin == 1) { 849 if (data_len == 1) { 850 bool updated = mptcp_update_rcv_data_fin(msk, mpext->data_seq, 851 mpext->dsn64); 852 pr_debug("DATA_FIN with no payload seq=%llu", mpext->data_seq); 853 if (subflow->map_valid) { 854 /* A DATA_FIN might arrive in a DSS 855 * option before the previous mapping 856 * has been fully consumed. Continue 857 * handling the existing mapping. 858 */ 859 skb_ext_del(skb, SKB_EXT_MPTCP); 860 return MAPPING_OK; 861 } else { 862 if (updated && schedule_work(&msk->work)) 863 sock_hold((struct sock *)msk); 864 865 return MAPPING_DATA_FIN; 866 } 867 } else { 868 u64 data_fin_seq = mpext->data_seq + data_len - 1; 869 870 /* If mpext->data_seq is a 32-bit value, data_fin_seq 871 * must also be limited to 32 bits. 872 */ 873 if (!mpext->dsn64) 874 data_fin_seq &= GENMASK_ULL(31, 0); 875 876 mptcp_update_rcv_data_fin(msk, data_fin_seq, mpext->dsn64); 877 pr_debug("DATA_FIN with mapping seq=%llu dsn64=%d", 878 data_fin_seq, mpext->dsn64); 879 } 880 881 /* Adjust for DATA_FIN using 1 byte of sequence space */ 882 data_len--; 883 } 884 885 if (!mpext->dsn64) { 886 map_seq = expand_seq(subflow->map_seq, subflow->map_data_len, 887 mpext->data_seq); 888 pr_debug("expanded seq=%llu", subflow->map_seq); 889 } else { 890 map_seq = mpext->data_seq; 891 } 892 WRITE_ONCE(mptcp_sk(subflow->conn)->use_64bit_ack, !!mpext->dsn64); 893 894 if (subflow->map_valid) { 895 /* Allow replacing only with an identical map */ 896 if (subflow->map_seq == map_seq && 897 subflow->map_subflow_seq == mpext->subflow_seq && 898 subflow->map_data_len == data_len) { 899 skb_ext_del(skb, SKB_EXT_MPTCP); 900 return MAPPING_OK; 901 } 902 903 /* If this skb data are fully covered by the current mapping, 904 * the new map would need caching, which is not supported 905 */ 906 if (skb_is_fully_mapped(ssk, skb)) { 907 MPTCP_INC_STATS(sock_net(ssk), MPTCP_MIB_DSSNOMATCH); 908 return MAPPING_INVALID; 909 } 910 911 /* will validate the next map after consuming the current one */ 912 return MAPPING_OK; 913 } 914 915 subflow->map_seq = map_seq; 916 subflow->map_subflow_seq = mpext->subflow_seq; 917 subflow->map_data_len = data_len; 918 subflow->map_valid = 1; 919 subflow->mpc_map = mpext->mpc_map; 920 pr_debug("new map seq=%llu subflow_seq=%u data_len=%u", 921 subflow->map_seq, subflow->map_subflow_seq, 922 subflow->map_data_len); 923 924 validate_seq: 925 /* we revalidate valid mapping on new skb, because we must ensure 926 * the current skb is completely covered by the available mapping 927 */ 928 if (!validate_mapping(ssk, skb)) 929 return MAPPING_INVALID; 930 931 skb_ext_del(skb, SKB_EXT_MPTCP); 932 return MAPPING_OK; 933 } 934 935 static void mptcp_subflow_discard_data(struct sock *ssk, struct sk_buff *skb, 936 u64 limit) 937 { 938 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk); 939 bool fin = TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN; 940 u32 incr; 941 942 incr = limit >= skb->len ? skb->len + fin : limit; 943 944 pr_debug("discarding=%d len=%d seq=%d", incr, skb->len, 945 subflow->map_subflow_seq); 946 MPTCP_INC_STATS(sock_net(ssk), MPTCP_MIB_DUPDATA); 947 tcp_sk(ssk)->copied_seq += incr; 948 if (!before(tcp_sk(ssk)->copied_seq, TCP_SKB_CB(skb)->end_seq)) 949 sk_eat_skb(ssk, skb); 950 if (mptcp_subflow_get_map_offset(subflow) >= subflow->map_data_len) 951 subflow->map_valid = 0; 952 } 953 954 /* sched mptcp worker to remove the subflow if no more data is pending */ 955 static void subflow_sched_work_if_closed(struct mptcp_sock *msk, struct sock *ssk) 956 { 957 struct sock *sk = (struct sock *)msk; 958 959 if (likely(ssk->sk_state != TCP_CLOSE)) 960 return; 961 962 if (skb_queue_empty(&ssk->sk_receive_queue) && 963 !test_and_set_bit(MPTCP_WORK_CLOSE_SUBFLOW, &msk->flags)) { 964 sock_hold(sk); 965 if (!schedule_work(&msk->work)) 966 sock_put(sk); 967 } 968 } 969 970 static bool subflow_check_data_avail(struct sock *ssk) 971 { 972 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk); 973 enum mapping_status status; 974 struct mptcp_sock *msk; 975 struct sk_buff *skb; 976 977 pr_debug("msk=%p ssk=%p data_avail=%d skb=%p", subflow->conn, ssk, 978 subflow->data_avail, skb_peek(&ssk->sk_receive_queue)); 979 if (!skb_peek(&ssk->sk_receive_queue)) 980 subflow->data_avail = 0; 981 if (subflow->data_avail) 982 return true; 983 984 msk = mptcp_sk(subflow->conn); 985 for (;;) { 986 u64 ack_seq; 987 u64 old_ack; 988 989 status = get_mapping_status(ssk, msk); 990 pr_debug("msk=%p ssk=%p status=%d", msk, ssk, status); 991 if (status == MAPPING_INVALID) { 992 ssk->sk_err = EBADMSG; 993 goto fatal; 994 } 995 if (status == MAPPING_DUMMY) { 996 __mptcp_do_fallback(msk); 997 skb = skb_peek(&ssk->sk_receive_queue); 998 subflow->map_valid = 1; 999 subflow->map_seq = READ_ONCE(msk->ack_seq); 1000 subflow->map_data_len = skb->len; 1001 subflow->map_subflow_seq = tcp_sk(ssk)->copied_seq - 1002 subflow->ssn_offset; 1003 subflow->data_avail = MPTCP_SUBFLOW_DATA_AVAIL; 1004 return true; 1005 } 1006 1007 if (status != MAPPING_OK) 1008 goto no_data; 1009 1010 skb = skb_peek(&ssk->sk_receive_queue); 1011 if (WARN_ON_ONCE(!skb)) 1012 goto no_data; 1013 1014 /* if msk lacks the remote key, this subflow must provide an 1015 * MP_CAPABLE-based mapping 1016 */ 1017 if (unlikely(!READ_ONCE(msk->can_ack))) { 1018 if (!subflow->mpc_map) { 1019 ssk->sk_err = EBADMSG; 1020 goto fatal; 1021 } 1022 WRITE_ONCE(msk->remote_key, subflow->remote_key); 1023 WRITE_ONCE(msk->ack_seq, subflow->map_seq); 1024 WRITE_ONCE(msk->can_ack, true); 1025 } 1026 1027 old_ack = READ_ONCE(msk->ack_seq); 1028 ack_seq = mptcp_subflow_get_mapped_dsn(subflow); 1029 pr_debug("msk ack_seq=%llx subflow ack_seq=%llx", old_ack, 1030 ack_seq); 1031 if (ack_seq == old_ack) { 1032 subflow->data_avail = MPTCP_SUBFLOW_DATA_AVAIL; 1033 break; 1034 } else if (after64(ack_seq, old_ack)) { 1035 subflow->data_avail = MPTCP_SUBFLOW_OOO_DATA; 1036 break; 1037 } 1038 1039 /* only accept in-sequence mapping. Old values are spurious 1040 * retransmission 1041 */ 1042 mptcp_subflow_discard_data(ssk, skb, old_ack - ack_seq); 1043 } 1044 return true; 1045 1046 no_data: 1047 subflow_sched_work_if_closed(msk, ssk); 1048 return false; 1049 fatal: 1050 /* fatal protocol error, close the socket */ 1051 /* This barrier is coupled with smp_rmb() in tcp_poll() */ 1052 smp_wmb(); 1053 ssk->sk_error_report(ssk); 1054 tcp_set_state(ssk, TCP_CLOSE); 1055 tcp_send_active_reset(ssk, GFP_ATOMIC); 1056 subflow->data_avail = 0; 1057 return false; 1058 } 1059 1060 bool mptcp_subflow_data_available(struct sock *sk) 1061 { 1062 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(sk); 1063 1064 /* check if current mapping is still valid */ 1065 if (subflow->map_valid && 1066 mptcp_subflow_get_map_offset(subflow) >= subflow->map_data_len) { 1067 subflow->map_valid = 0; 1068 subflow->data_avail = 0; 1069 1070 pr_debug("Done with mapping: seq=%u data_len=%u", 1071 subflow->map_subflow_seq, 1072 subflow->map_data_len); 1073 } 1074 1075 return subflow_check_data_avail(sk); 1076 } 1077 1078 /* If ssk has an mptcp parent socket, use the mptcp rcvbuf occupancy, 1079 * not the ssk one. 1080 * 1081 * In mptcp, rwin is about the mptcp-level connection data. 1082 * 1083 * Data that is still on the ssk rx queue can thus be ignored, 1084 * as far as mptcp peer is concerened that data is still inflight. 1085 * DSS ACK is updated when skb is moved to the mptcp rx queue. 1086 */ 1087 void mptcp_space(const struct sock *ssk, int *space, int *full_space) 1088 { 1089 const struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk); 1090 const struct sock *sk = subflow->conn; 1091 1092 *space = __mptcp_space(sk); 1093 *full_space = tcp_full_space(sk); 1094 } 1095 1096 static void subflow_data_ready(struct sock *sk) 1097 { 1098 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(sk); 1099 u16 state = 1 << inet_sk_state_load(sk); 1100 struct sock *parent = subflow->conn; 1101 struct mptcp_sock *msk; 1102 1103 msk = mptcp_sk(parent); 1104 if (state & TCPF_LISTEN) { 1105 /* MPJ subflow are removed from accept queue before reaching here, 1106 * avoid stray wakeups 1107 */ 1108 if (reqsk_queue_empty(&inet_csk(sk)->icsk_accept_queue)) 1109 return; 1110 1111 set_bit(MPTCP_DATA_READY, &msk->flags); 1112 parent->sk_data_ready(parent); 1113 return; 1114 } 1115 1116 WARN_ON_ONCE(!__mptcp_check_fallback(msk) && !subflow->mp_capable && 1117 !subflow->mp_join && !(state & TCPF_CLOSE)); 1118 1119 if (mptcp_subflow_data_available(sk)) 1120 mptcp_data_ready(parent, sk); 1121 } 1122 1123 static void subflow_write_space(struct sock *ssk) 1124 { 1125 struct sock *sk = mptcp_subflow_ctx(ssk)->conn; 1126 1127 mptcp_propagate_sndbuf(sk, ssk); 1128 mptcp_write_space(sk); 1129 } 1130 1131 void __mptcp_error_report(struct sock *sk) 1132 { 1133 struct mptcp_subflow_context *subflow; 1134 struct mptcp_sock *msk = mptcp_sk(sk); 1135 1136 mptcp_for_each_subflow(msk, subflow) { 1137 struct sock *ssk = mptcp_subflow_tcp_sock(subflow); 1138 int err = sock_error(ssk); 1139 1140 if (!err) 1141 continue; 1142 1143 /* only propagate errors on fallen-back sockets or 1144 * on MPC connect 1145 */ 1146 if (sk->sk_state != TCP_SYN_SENT && !__mptcp_check_fallback(msk)) 1147 continue; 1148 1149 inet_sk_state_store(sk, inet_sk_state_load(ssk)); 1150 sk->sk_err = -err; 1151 1152 /* This barrier is coupled with smp_rmb() in mptcp_poll() */ 1153 smp_wmb(); 1154 sk->sk_error_report(sk); 1155 break; 1156 } 1157 } 1158 1159 static void subflow_error_report(struct sock *ssk) 1160 { 1161 struct sock *sk = mptcp_subflow_ctx(ssk)->conn; 1162 1163 mptcp_data_lock(sk); 1164 if (!sock_owned_by_user(sk)) 1165 __mptcp_error_report(sk); 1166 else 1167 set_bit(MPTCP_ERROR_REPORT, &mptcp_sk(sk)->flags); 1168 mptcp_data_unlock(sk); 1169 } 1170 1171 static struct inet_connection_sock_af_ops * 1172 subflow_default_af_ops(struct sock *sk) 1173 { 1174 #if IS_ENABLED(CONFIG_MPTCP_IPV6) 1175 if (sk->sk_family == AF_INET6) 1176 return &subflow_v6_specific; 1177 #endif 1178 return &subflow_specific; 1179 } 1180 1181 #if IS_ENABLED(CONFIG_MPTCP_IPV6) 1182 void mptcpv6_handle_mapped(struct sock *sk, bool mapped) 1183 { 1184 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(sk); 1185 struct inet_connection_sock *icsk = inet_csk(sk); 1186 struct inet_connection_sock_af_ops *target; 1187 1188 target = mapped ? &subflow_v6m_specific : subflow_default_af_ops(sk); 1189 1190 pr_debug("subflow=%p family=%d ops=%p target=%p mapped=%d", 1191 subflow, sk->sk_family, icsk->icsk_af_ops, target, mapped); 1192 1193 if (likely(icsk->icsk_af_ops == target)) 1194 return; 1195 1196 subflow->icsk_af_ops = icsk->icsk_af_ops; 1197 icsk->icsk_af_ops = target; 1198 } 1199 #endif 1200 1201 void mptcp_info2sockaddr(const struct mptcp_addr_info *info, 1202 struct sockaddr_storage *addr, 1203 unsigned short family) 1204 { 1205 memset(addr, 0, sizeof(*addr)); 1206 addr->ss_family = family; 1207 if (addr->ss_family == AF_INET) { 1208 struct sockaddr_in *in_addr = (struct sockaddr_in *)addr; 1209 1210 if (info->family == AF_INET) 1211 in_addr->sin_addr = info->addr; 1212 #if IS_ENABLED(CONFIG_MPTCP_IPV6) 1213 else if (ipv6_addr_v4mapped(&info->addr6)) 1214 in_addr->sin_addr.s_addr = info->addr6.s6_addr32[3]; 1215 #endif 1216 in_addr->sin_port = info->port; 1217 } 1218 #if IS_ENABLED(CONFIG_MPTCP_IPV6) 1219 else if (addr->ss_family == AF_INET6) { 1220 struct sockaddr_in6 *in6_addr = (struct sockaddr_in6 *)addr; 1221 1222 if (info->family == AF_INET) 1223 ipv6_addr_set_v4mapped(info->addr.s_addr, 1224 &in6_addr->sin6_addr); 1225 else 1226 in6_addr->sin6_addr = info->addr6; 1227 in6_addr->sin6_port = info->port; 1228 } 1229 #endif 1230 } 1231 1232 int __mptcp_subflow_connect(struct sock *sk, const struct mptcp_addr_info *loc, 1233 const struct mptcp_addr_info *remote) 1234 { 1235 struct mptcp_sock *msk = mptcp_sk(sk); 1236 struct mptcp_subflow_context *subflow; 1237 struct sockaddr_storage addr; 1238 int remote_id = remote->id; 1239 int local_id = loc->id; 1240 struct socket *sf; 1241 struct sock *ssk; 1242 u32 remote_token; 1243 int addrlen; 1244 int err; 1245 1246 if (!mptcp_is_fully_established(sk)) 1247 return -ENOTCONN; 1248 1249 err = mptcp_subflow_create_socket(sk, &sf); 1250 if (err) 1251 return err; 1252 1253 ssk = sf->sk; 1254 subflow = mptcp_subflow_ctx(ssk); 1255 do { 1256 get_random_bytes(&subflow->local_nonce, sizeof(u32)); 1257 } while (!subflow->local_nonce); 1258 1259 if (!local_id) { 1260 err = mptcp_pm_get_local_id(msk, (struct sock_common *)ssk); 1261 if (err < 0) 1262 goto failed; 1263 1264 local_id = err; 1265 } 1266 1267 subflow->remote_key = msk->remote_key; 1268 subflow->local_key = msk->local_key; 1269 subflow->token = msk->token; 1270 mptcp_info2sockaddr(loc, &addr, ssk->sk_family); 1271 1272 addrlen = sizeof(struct sockaddr_in); 1273 #if IS_ENABLED(CONFIG_MPTCP_IPV6) 1274 if (addr.ss_family == AF_INET6) 1275 addrlen = sizeof(struct sockaddr_in6); 1276 #endif 1277 ssk->sk_bound_dev_if = loc->ifindex; 1278 err = kernel_bind(sf, (struct sockaddr *)&addr, addrlen); 1279 if (err) 1280 goto failed; 1281 1282 mptcp_crypto_key_sha(subflow->remote_key, &remote_token, NULL); 1283 pr_debug("msk=%p remote_token=%u local_id=%d remote_id=%d", msk, 1284 remote_token, local_id, remote_id); 1285 subflow->remote_token = remote_token; 1286 subflow->local_id = local_id; 1287 subflow->remote_id = remote_id; 1288 subflow->request_join = 1; 1289 subflow->request_bkup = !!(loc->flags & MPTCP_PM_ADDR_FLAG_BACKUP); 1290 mptcp_info2sockaddr(remote, &addr, ssk->sk_family); 1291 1292 mptcp_add_pending_subflow(msk, subflow); 1293 err = kernel_connect(sf, (struct sockaddr *)&addr, addrlen, O_NONBLOCK); 1294 if (err && err != -EINPROGRESS) 1295 goto failed_unlink; 1296 1297 /* discard the subflow socket */ 1298 mptcp_sock_graft(ssk, sk->sk_socket); 1299 iput(SOCK_INODE(sf)); 1300 return err; 1301 1302 failed_unlink: 1303 spin_lock_bh(&msk->join_list_lock); 1304 list_del(&subflow->node); 1305 spin_unlock_bh(&msk->join_list_lock); 1306 sock_put(mptcp_subflow_tcp_sock(subflow)); 1307 1308 failed: 1309 subflow->disposable = 1; 1310 sock_release(sf); 1311 return err; 1312 } 1313 1314 static void mptcp_attach_cgroup(struct sock *parent, struct sock *child) 1315 { 1316 #ifdef CONFIG_SOCK_CGROUP_DATA 1317 struct sock_cgroup_data *parent_skcd = &parent->sk_cgrp_data, 1318 *child_skcd = &child->sk_cgrp_data; 1319 1320 /* only the additional subflows created by kworkers have to be modified */ 1321 if (cgroup_id(sock_cgroup_ptr(parent_skcd)) != 1322 cgroup_id(sock_cgroup_ptr(child_skcd))) { 1323 #ifdef CONFIG_MEMCG 1324 struct mem_cgroup *memcg = parent->sk_memcg; 1325 1326 mem_cgroup_sk_free(child); 1327 if (memcg && css_tryget(&memcg->css)) 1328 child->sk_memcg = memcg; 1329 #endif /* CONFIG_MEMCG */ 1330 1331 cgroup_sk_free(child_skcd); 1332 *child_skcd = *parent_skcd; 1333 cgroup_sk_clone(child_skcd); 1334 } 1335 #endif /* CONFIG_SOCK_CGROUP_DATA */ 1336 } 1337 1338 static void mptcp_subflow_ops_override(struct sock *ssk) 1339 { 1340 #if IS_ENABLED(CONFIG_MPTCP_IPV6) 1341 if (ssk->sk_prot == &tcpv6_prot) 1342 ssk->sk_prot = &tcpv6_prot_override; 1343 else 1344 #endif 1345 ssk->sk_prot = &tcp_prot_override; 1346 } 1347 1348 static void mptcp_subflow_ops_undo_override(struct sock *ssk) 1349 { 1350 #if IS_ENABLED(CONFIG_MPTCP_IPV6) 1351 if (ssk->sk_prot == &tcpv6_prot_override) 1352 ssk->sk_prot = &tcpv6_prot; 1353 else 1354 #endif 1355 ssk->sk_prot = &tcp_prot; 1356 } 1357 int mptcp_subflow_create_socket(struct sock *sk, struct socket **new_sock) 1358 { 1359 struct mptcp_subflow_context *subflow; 1360 struct net *net = sock_net(sk); 1361 struct socket *sf; 1362 int err; 1363 1364 /* un-accepted server sockets can reach here - on bad configuration 1365 * bail early to avoid greater trouble later 1366 */ 1367 if (unlikely(!sk->sk_socket)) 1368 return -EINVAL; 1369 1370 err = sock_create_kern(net, sk->sk_family, SOCK_STREAM, IPPROTO_TCP, 1371 &sf); 1372 if (err) 1373 return err; 1374 1375 lock_sock(sf->sk); 1376 1377 /* the newly created socket has to be in the same cgroup as its parent */ 1378 mptcp_attach_cgroup(sk, sf->sk); 1379 1380 /* kernel sockets do not by default acquire net ref, but TCP timer 1381 * needs it. 1382 */ 1383 sf->sk->sk_net_refcnt = 1; 1384 get_net(net); 1385 #ifdef CONFIG_PROC_FS 1386 this_cpu_add(*net->core.sock_inuse, 1); 1387 #endif 1388 err = tcp_set_ulp(sf->sk, "mptcp"); 1389 release_sock(sf->sk); 1390 1391 if (err) { 1392 sock_release(sf); 1393 return err; 1394 } 1395 1396 /* the newly created socket really belongs to the owning MPTCP master 1397 * socket, even if for additional subflows the allocation is performed 1398 * by a kernel workqueue. Adjust inode references, so that the 1399 * procfs/diag interaces really show this one belonging to the correct 1400 * user. 1401 */ 1402 SOCK_INODE(sf)->i_ino = SOCK_INODE(sk->sk_socket)->i_ino; 1403 SOCK_INODE(sf)->i_uid = SOCK_INODE(sk->sk_socket)->i_uid; 1404 SOCK_INODE(sf)->i_gid = SOCK_INODE(sk->sk_socket)->i_gid; 1405 1406 subflow = mptcp_subflow_ctx(sf->sk); 1407 pr_debug("subflow=%p", subflow); 1408 1409 *new_sock = sf; 1410 sock_hold(sk); 1411 subflow->conn = sk; 1412 mptcp_subflow_ops_override(sf->sk); 1413 1414 return 0; 1415 } 1416 1417 static struct mptcp_subflow_context *subflow_create_ctx(struct sock *sk, 1418 gfp_t priority) 1419 { 1420 struct inet_connection_sock *icsk = inet_csk(sk); 1421 struct mptcp_subflow_context *ctx; 1422 1423 ctx = kzalloc(sizeof(*ctx), priority); 1424 if (!ctx) 1425 return NULL; 1426 1427 rcu_assign_pointer(icsk->icsk_ulp_data, ctx); 1428 INIT_LIST_HEAD(&ctx->node); 1429 INIT_LIST_HEAD(&ctx->delegated_node); 1430 1431 pr_debug("subflow=%p", ctx); 1432 1433 ctx->tcp_sock = sk; 1434 1435 return ctx; 1436 } 1437 1438 static void __subflow_state_change(struct sock *sk) 1439 { 1440 struct socket_wq *wq; 1441 1442 rcu_read_lock(); 1443 wq = rcu_dereference(sk->sk_wq); 1444 if (skwq_has_sleeper(wq)) 1445 wake_up_interruptible_all(&wq->wait); 1446 rcu_read_unlock(); 1447 } 1448 1449 static bool subflow_is_done(const struct sock *sk) 1450 { 1451 return sk->sk_shutdown & RCV_SHUTDOWN || sk->sk_state == TCP_CLOSE; 1452 } 1453 1454 static void subflow_state_change(struct sock *sk) 1455 { 1456 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(sk); 1457 struct sock *parent = subflow->conn; 1458 1459 __subflow_state_change(sk); 1460 1461 if (subflow_simultaneous_connect(sk)) { 1462 mptcp_propagate_sndbuf(parent, sk); 1463 mptcp_do_fallback(sk); 1464 mptcp_rcv_space_init(mptcp_sk(parent), sk); 1465 pr_fallback(mptcp_sk(parent)); 1466 subflow->conn_finished = 1; 1467 if (inet_sk_state_load(parent) == TCP_SYN_SENT) { 1468 inet_sk_state_store(parent, TCP_ESTABLISHED); 1469 parent->sk_state_change(parent); 1470 } 1471 } 1472 1473 /* as recvmsg() does not acquire the subflow socket for ssk selection 1474 * a fin packet carrying a DSS can be unnoticed if we don't trigger 1475 * the data available machinery here. 1476 */ 1477 if (mptcp_subflow_data_available(sk)) 1478 mptcp_data_ready(parent, sk); 1479 1480 subflow_sched_work_if_closed(mptcp_sk(parent), sk); 1481 1482 if (__mptcp_check_fallback(mptcp_sk(parent)) && 1483 !subflow->rx_eof && subflow_is_done(sk)) { 1484 subflow->rx_eof = 1; 1485 mptcp_subflow_eof(parent); 1486 } 1487 } 1488 1489 static int subflow_ulp_init(struct sock *sk) 1490 { 1491 struct inet_connection_sock *icsk = inet_csk(sk); 1492 struct mptcp_subflow_context *ctx; 1493 struct tcp_sock *tp = tcp_sk(sk); 1494 int err = 0; 1495 1496 /* disallow attaching ULP to a socket unless it has been 1497 * created with sock_create_kern() 1498 */ 1499 if (!sk->sk_kern_sock) { 1500 err = -EOPNOTSUPP; 1501 goto out; 1502 } 1503 1504 ctx = subflow_create_ctx(sk, GFP_KERNEL); 1505 if (!ctx) { 1506 err = -ENOMEM; 1507 goto out; 1508 } 1509 1510 pr_debug("subflow=%p, family=%d", ctx, sk->sk_family); 1511 1512 tp->is_mptcp = 1; 1513 ctx->icsk_af_ops = icsk->icsk_af_ops; 1514 icsk->icsk_af_ops = subflow_default_af_ops(sk); 1515 ctx->tcp_data_ready = sk->sk_data_ready; 1516 ctx->tcp_state_change = sk->sk_state_change; 1517 ctx->tcp_write_space = sk->sk_write_space; 1518 ctx->tcp_error_report = sk->sk_error_report; 1519 sk->sk_data_ready = subflow_data_ready; 1520 sk->sk_write_space = subflow_write_space; 1521 sk->sk_state_change = subflow_state_change; 1522 sk->sk_error_report = subflow_error_report; 1523 out: 1524 return err; 1525 } 1526 1527 static void subflow_ulp_release(struct sock *ssk) 1528 { 1529 struct mptcp_subflow_context *ctx = mptcp_subflow_ctx(ssk); 1530 bool release = true; 1531 struct sock *sk; 1532 1533 if (!ctx) 1534 return; 1535 1536 sk = ctx->conn; 1537 if (sk) { 1538 /* if the msk has been orphaned, keep the ctx 1539 * alive, will be freed by __mptcp_close_ssk(), 1540 * when the subflow is still unaccepted 1541 */ 1542 release = ctx->disposable || list_empty(&ctx->node); 1543 sock_put(sk); 1544 } 1545 1546 mptcp_subflow_ops_undo_override(ssk); 1547 if (release) 1548 kfree_rcu(ctx, rcu); 1549 } 1550 1551 static void subflow_ulp_clone(const struct request_sock *req, 1552 struct sock *newsk, 1553 const gfp_t priority) 1554 { 1555 struct mptcp_subflow_request_sock *subflow_req = mptcp_subflow_rsk(req); 1556 struct mptcp_subflow_context *old_ctx = mptcp_subflow_ctx(newsk); 1557 struct mptcp_subflow_context *new_ctx; 1558 1559 if (!tcp_rsk(req)->is_mptcp || 1560 (!subflow_req->mp_capable && !subflow_req->mp_join)) { 1561 subflow_ulp_fallback(newsk, old_ctx); 1562 return; 1563 } 1564 1565 new_ctx = subflow_create_ctx(newsk, priority); 1566 if (!new_ctx) { 1567 subflow_ulp_fallback(newsk, old_ctx); 1568 return; 1569 } 1570 1571 new_ctx->conn_finished = 1; 1572 new_ctx->icsk_af_ops = old_ctx->icsk_af_ops; 1573 new_ctx->tcp_data_ready = old_ctx->tcp_data_ready; 1574 new_ctx->tcp_state_change = old_ctx->tcp_state_change; 1575 new_ctx->tcp_write_space = old_ctx->tcp_write_space; 1576 new_ctx->tcp_error_report = old_ctx->tcp_error_report; 1577 new_ctx->rel_write_seq = 1; 1578 new_ctx->tcp_sock = newsk; 1579 1580 if (subflow_req->mp_capable) { 1581 /* see comments in subflow_syn_recv_sock(), MPTCP connection 1582 * is fully established only after we receive the remote key 1583 */ 1584 new_ctx->mp_capable = 1; 1585 new_ctx->local_key = subflow_req->local_key; 1586 new_ctx->token = subflow_req->token; 1587 new_ctx->ssn_offset = subflow_req->ssn_offset; 1588 new_ctx->idsn = subflow_req->idsn; 1589 } else if (subflow_req->mp_join) { 1590 new_ctx->ssn_offset = subflow_req->ssn_offset; 1591 new_ctx->mp_join = 1; 1592 new_ctx->fully_established = 1; 1593 new_ctx->backup = subflow_req->backup; 1594 new_ctx->local_id = subflow_req->local_id; 1595 new_ctx->remote_id = subflow_req->remote_id; 1596 new_ctx->token = subflow_req->token; 1597 new_ctx->thmac = subflow_req->thmac; 1598 } 1599 } 1600 1601 static void tcp_release_cb_override(struct sock *ssk) 1602 { 1603 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk); 1604 1605 if (mptcp_subflow_has_delegated_action(subflow)) 1606 mptcp_subflow_process_delegated(ssk); 1607 1608 tcp_release_cb(ssk); 1609 } 1610 1611 static struct tcp_ulp_ops subflow_ulp_ops __read_mostly = { 1612 .name = "mptcp", 1613 .owner = THIS_MODULE, 1614 .init = subflow_ulp_init, 1615 .release = subflow_ulp_release, 1616 .clone = subflow_ulp_clone, 1617 }; 1618 1619 static int subflow_ops_init(struct request_sock_ops *subflow_ops) 1620 { 1621 subflow_ops->obj_size = sizeof(struct mptcp_subflow_request_sock); 1622 subflow_ops->slab_name = "request_sock_subflow"; 1623 1624 subflow_ops->slab = kmem_cache_create(subflow_ops->slab_name, 1625 subflow_ops->obj_size, 0, 1626 SLAB_ACCOUNT | 1627 SLAB_TYPESAFE_BY_RCU, 1628 NULL); 1629 if (!subflow_ops->slab) 1630 return -ENOMEM; 1631 1632 subflow_ops->destructor = subflow_req_destructor; 1633 1634 return 0; 1635 } 1636 1637 void __init mptcp_subflow_init(void) 1638 { 1639 mptcp_subflow_request_sock_ops = tcp_request_sock_ops; 1640 if (subflow_ops_init(&mptcp_subflow_request_sock_ops) != 0) 1641 panic("MPTCP: failed to init subflow request sock ops\n"); 1642 1643 subflow_request_sock_ipv4_ops = tcp_request_sock_ipv4_ops; 1644 subflow_request_sock_ipv4_ops.route_req = subflow_v4_route_req; 1645 1646 subflow_specific = ipv4_specific; 1647 subflow_specific.conn_request = subflow_v4_conn_request; 1648 subflow_specific.syn_recv_sock = subflow_syn_recv_sock; 1649 subflow_specific.sk_rx_dst_set = subflow_finish_connect; 1650 1651 tcp_prot_override = tcp_prot; 1652 tcp_prot_override.release_cb = tcp_release_cb_override; 1653 1654 #if IS_ENABLED(CONFIG_MPTCP_IPV6) 1655 subflow_request_sock_ipv6_ops = tcp_request_sock_ipv6_ops; 1656 subflow_request_sock_ipv6_ops.route_req = subflow_v6_route_req; 1657 1658 subflow_v6_specific = ipv6_specific; 1659 subflow_v6_specific.conn_request = subflow_v6_conn_request; 1660 subflow_v6_specific.syn_recv_sock = subflow_syn_recv_sock; 1661 subflow_v6_specific.sk_rx_dst_set = subflow_finish_connect; 1662 1663 subflow_v6m_specific = subflow_v6_specific; 1664 subflow_v6m_specific.queue_xmit = ipv4_specific.queue_xmit; 1665 subflow_v6m_specific.send_check = ipv4_specific.send_check; 1666 subflow_v6m_specific.net_header_len = ipv4_specific.net_header_len; 1667 subflow_v6m_specific.mtu_reduced = ipv4_specific.mtu_reduced; 1668 subflow_v6m_specific.net_frag_header_len = 0; 1669 1670 tcpv6_prot_override = tcpv6_prot; 1671 tcpv6_prot_override.release_cb = tcp_release_cb_override; 1672 #endif 1673 1674 mptcp_diag_subflow_init(&subflow_ulp_ops); 1675 1676 if (tcp_register_ulp(&subflow_ulp_ops) != 0) 1677 panic("MPTCP: failed to register subflows to ULP\n"); 1678 } 1679