1 // SPDX-License-Identifier: GPL-2.0 2 /* Multipath TCP 3 * 4 * Copyright (c) 2017 - 2019, Intel Corporation. 5 */ 6 7 #define pr_fmt(fmt) "MPTCP: " fmt 8 9 #include <linux/kernel.h> 10 #include <linux/module.h> 11 #include <linux/netdevice.h> 12 #include <crypto/algapi.h> 13 #include <crypto/sha2.h> 14 #include <net/sock.h> 15 #include <net/inet_common.h> 16 #include <net/inet_hashtables.h> 17 #include <net/protocol.h> 18 #include <net/tcp.h> 19 #if IS_ENABLED(CONFIG_MPTCP_IPV6) 20 #include <net/ip6_route.h> 21 #include <net/transp_v6.h> 22 #endif 23 #include <net/mptcp.h> 24 #include <uapi/linux/mptcp.h> 25 #include "protocol.h" 26 #include "mib.h" 27 28 #include <trace/events/mptcp.h> 29 #include <trace/events/sock.h> 30 31 static void mptcp_subflow_ops_undo_override(struct sock *ssk); 32 33 static void SUBFLOW_REQ_INC_STATS(struct request_sock *req, 34 enum linux_mptcp_mib_field field) 35 { 36 MPTCP_INC_STATS(sock_net(req_to_sk(req)), field); 37 } 38 39 static void subflow_req_destructor(struct request_sock *req) 40 { 41 struct mptcp_subflow_request_sock *subflow_req = mptcp_subflow_rsk(req); 42 43 pr_debug("subflow_req=%p", subflow_req); 44 45 if (subflow_req->msk) 46 sock_put((struct sock *)subflow_req->msk); 47 48 mptcp_token_destroy_request(req); 49 } 50 51 static void subflow_generate_hmac(u64 key1, u64 key2, u32 nonce1, u32 nonce2, 52 void *hmac) 53 { 54 u8 msg[8]; 55 56 put_unaligned_be32(nonce1, &msg[0]); 57 put_unaligned_be32(nonce2, &msg[4]); 58 59 mptcp_crypto_hmac_sha(key1, key2, msg, 8, hmac); 60 } 61 62 static bool mptcp_can_accept_new_subflow(const struct mptcp_sock *msk) 63 { 64 return mptcp_is_fully_established((void *)msk) && 65 ((mptcp_pm_is_userspace(msk) && 66 mptcp_userspace_pm_active(msk)) || 67 READ_ONCE(msk->pm.accept_subflow)); 68 } 69 70 /* validate received token and create truncated hmac and nonce for SYN-ACK */ 71 static void subflow_req_create_thmac(struct mptcp_subflow_request_sock *subflow_req) 72 { 73 struct mptcp_sock *msk = subflow_req->msk; 74 u8 hmac[SHA256_DIGEST_SIZE]; 75 76 get_random_bytes(&subflow_req->local_nonce, sizeof(u32)); 77 78 subflow_generate_hmac(msk->local_key, msk->remote_key, 79 subflow_req->local_nonce, 80 subflow_req->remote_nonce, hmac); 81 82 subflow_req->thmac = get_unaligned_be64(hmac); 83 } 84 85 static struct mptcp_sock *subflow_token_join_request(struct request_sock *req) 86 { 87 struct mptcp_subflow_request_sock *subflow_req = mptcp_subflow_rsk(req); 88 struct mptcp_sock *msk; 89 int local_id; 90 91 msk = mptcp_token_get_sock(sock_net(req_to_sk(req)), subflow_req->token); 92 if (!msk) { 93 SUBFLOW_REQ_INC_STATS(req, MPTCP_MIB_JOINNOTOKEN); 94 return NULL; 95 } 96 97 local_id = mptcp_pm_get_local_id(msk, (struct sock_common *)req); 98 if (local_id < 0) { 99 sock_put((struct sock *)msk); 100 return NULL; 101 } 102 subflow_req->local_id = local_id; 103 104 return msk; 105 } 106 107 static void subflow_init_req(struct request_sock *req, const struct sock *sk_listener) 108 { 109 struct mptcp_subflow_request_sock *subflow_req = mptcp_subflow_rsk(req); 110 111 subflow_req->mp_capable = 0; 112 subflow_req->mp_join = 0; 113 subflow_req->csum_reqd = mptcp_is_checksum_enabled(sock_net(sk_listener)); 114 subflow_req->allow_join_id0 = mptcp_allow_join_id0(sock_net(sk_listener)); 115 subflow_req->msk = NULL; 116 mptcp_token_init_request(req); 117 } 118 119 static bool subflow_use_different_sport(struct mptcp_sock *msk, const struct sock *sk) 120 { 121 return inet_sk(sk)->inet_sport != inet_sk((struct sock *)msk)->inet_sport; 122 } 123 124 static void subflow_add_reset_reason(struct sk_buff *skb, u8 reason) 125 { 126 struct mptcp_ext *mpext = skb_ext_add(skb, SKB_EXT_MPTCP); 127 128 if (mpext) { 129 memset(mpext, 0, sizeof(*mpext)); 130 mpext->reset_reason = reason; 131 } 132 } 133 134 /* Init mptcp request socket. 135 * 136 * Returns an error code if a JOIN has failed and a TCP reset 137 * should be sent. 138 */ 139 static int subflow_check_req(struct request_sock *req, 140 const struct sock *sk_listener, 141 struct sk_buff *skb) 142 { 143 struct mptcp_subflow_context *listener = mptcp_subflow_ctx(sk_listener); 144 struct mptcp_subflow_request_sock *subflow_req = mptcp_subflow_rsk(req); 145 struct mptcp_options_received mp_opt; 146 bool opt_mp_capable, opt_mp_join; 147 148 pr_debug("subflow_req=%p, listener=%p", subflow_req, listener); 149 150 #ifdef CONFIG_TCP_MD5SIG 151 /* no MPTCP if MD5SIG is enabled on this socket or we may run out of 152 * TCP option space. 153 */ 154 if (rcu_access_pointer(tcp_sk(sk_listener)->md5sig_info)) 155 return -EINVAL; 156 #endif 157 158 mptcp_get_options(skb, &mp_opt); 159 160 opt_mp_capable = !!(mp_opt.suboptions & OPTIONS_MPTCP_MPC); 161 opt_mp_join = !!(mp_opt.suboptions & OPTIONS_MPTCP_MPJ); 162 if (opt_mp_capable) { 163 SUBFLOW_REQ_INC_STATS(req, MPTCP_MIB_MPCAPABLEPASSIVE); 164 165 if (opt_mp_join) 166 return 0; 167 } else if (opt_mp_join) { 168 SUBFLOW_REQ_INC_STATS(req, MPTCP_MIB_JOINSYNRX); 169 } 170 171 if (opt_mp_capable && listener->request_mptcp) { 172 int err, retries = MPTCP_TOKEN_MAX_RETRIES; 173 174 subflow_req->ssn_offset = TCP_SKB_CB(skb)->seq; 175 again: 176 do { 177 get_random_bytes(&subflow_req->local_key, sizeof(subflow_req->local_key)); 178 } while (subflow_req->local_key == 0); 179 180 if (unlikely(req->syncookie)) { 181 mptcp_crypto_key_sha(subflow_req->local_key, 182 &subflow_req->token, 183 &subflow_req->idsn); 184 if (mptcp_token_exists(subflow_req->token)) { 185 if (retries-- > 0) 186 goto again; 187 SUBFLOW_REQ_INC_STATS(req, MPTCP_MIB_TOKENFALLBACKINIT); 188 } else { 189 subflow_req->mp_capable = 1; 190 } 191 return 0; 192 } 193 194 err = mptcp_token_new_request(req); 195 if (err == 0) 196 subflow_req->mp_capable = 1; 197 else if (retries-- > 0) 198 goto again; 199 else 200 SUBFLOW_REQ_INC_STATS(req, MPTCP_MIB_TOKENFALLBACKINIT); 201 202 } else if (opt_mp_join && listener->request_mptcp) { 203 subflow_req->ssn_offset = TCP_SKB_CB(skb)->seq; 204 subflow_req->mp_join = 1; 205 subflow_req->backup = mp_opt.backup; 206 subflow_req->remote_id = mp_opt.join_id; 207 subflow_req->token = mp_opt.token; 208 subflow_req->remote_nonce = mp_opt.nonce; 209 subflow_req->msk = subflow_token_join_request(req); 210 211 /* Can't fall back to TCP in this case. */ 212 if (!subflow_req->msk) { 213 subflow_add_reset_reason(skb, MPTCP_RST_EMPTCP); 214 return -EPERM; 215 } 216 217 if (subflow_use_different_sport(subflow_req->msk, sk_listener)) { 218 pr_debug("syn inet_sport=%d %d", 219 ntohs(inet_sk(sk_listener)->inet_sport), 220 ntohs(inet_sk((struct sock *)subflow_req->msk)->inet_sport)); 221 if (!mptcp_pm_sport_in_anno_list(subflow_req->msk, sk_listener)) { 222 SUBFLOW_REQ_INC_STATS(req, MPTCP_MIB_MISMATCHPORTSYNRX); 223 return -EPERM; 224 } 225 SUBFLOW_REQ_INC_STATS(req, MPTCP_MIB_JOINPORTSYNRX); 226 } 227 228 subflow_req_create_thmac(subflow_req); 229 230 if (unlikely(req->syncookie)) { 231 if (mptcp_can_accept_new_subflow(subflow_req->msk)) 232 subflow_init_req_cookie_join_save(subflow_req, skb); 233 else 234 return -EPERM; 235 } 236 237 pr_debug("token=%u, remote_nonce=%u msk=%p", subflow_req->token, 238 subflow_req->remote_nonce, subflow_req->msk); 239 } 240 241 return 0; 242 } 243 244 int mptcp_subflow_init_cookie_req(struct request_sock *req, 245 const struct sock *sk_listener, 246 struct sk_buff *skb) 247 { 248 struct mptcp_subflow_context *listener = mptcp_subflow_ctx(sk_listener); 249 struct mptcp_subflow_request_sock *subflow_req = mptcp_subflow_rsk(req); 250 struct mptcp_options_received mp_opt; 251 bool opt_mp_capable, opt_mp_join; 252 int err; 253 254 subflow_init_req(req, sk_listener); 255 mptcp_get_options(skb, &mp_opt); 256 257 opt_mp_capable = !!(mp_opt.suboptions & OPTIONS_MPTCP_MPC); 258 opt_mp_join = !!(mp_opt.suboptions & OPTIONS_MPTCP_MPJ); 259 if (opt_mp_capable && opt_mp_join) 260 return -EINVAL; 261 262 if (opt_mp_capable && listener->request_mptcp) { 263 if (mp_opt.sndr_key == 0) 264 return -EINVAL; 265 266 subflow_req->local_key = mp_opt.rcvr_key; 267 err = mptcp_token_new_request(req); 268 if (err) 269 return err; 270 271 subflow_req->mp_capable = 1; 272 subflow_req->ssn_offset = TCP_SKB_CB(skb)->seq - 1; 273 } else if (opt_mp_join && listener->request_mptcp) { 274 if (!mptcp_token_join_cookie_init_state(subflow_req, skb)) 275 return -EINVAL; 276 277 subflow_req->mp_join = 1; 278 subflow_req->ssn_offset = TCP_SKB_CB(skb)->seq - 1; 279 } 280 281 return 0; 282 } 283 EXPORT_SYMBOL_GPL(mptcp_subflow_init_cookie_req); 284 285 static struct dst_entry *subflow_v4_route_req(const struct sock *sk, 286 struct sk_buff *skb, 287 struct flowi *fl, 288 struct request_sock *req) 289 { 290 struct dst_entry *dst; 291 int err; 292 293 tcp_rsk(req)->is_mptcp = 1; 294 subflow_init_req(req, sk); 295 296 dst = tcp_request_sock_ipv4_ops.route_req(sk, skb, fl, req); 297 if (!dst) 298 return NULL; 299 300 err = subflow_check_req(req, sk, skb); 301 if (err == 0) 302 return dst; 303 304 dst_release(dst); 305 if (!req->syncookie) 306 tcp_request_sock_ops.send_reset(sk, skb); 307 return NULL; 308 } 309 310 static void subflow_prep_synack(const struct sock *sk, struct request_sock *req, 311 struct tcp_fastopen_cookie *foc, 312 enum tcp_synack_type synack_type) 313 { 314 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(sk); 315 struct inet_request_sock *ireq = inet_rsk(req); 316 317 /* clear tstamp_ok, as needed depending on cookie */ 318 if (foc && foc->len > -1) 319 ireq->tstamp_ok = 0; 320 321 if (synack_type == TCP_SYNACK_FASTOPEN) 322 mptcp_fastopen_subflow_synack_set_params(subflow, req); 323 } 324 325 static int subflow_v4_send_synack(const struct sock *sk, struct dst_entry *dst, 326 struct flowi *fl, 327 struct request_sock *req, 328 struct tcp_fastopen_cookie *foc, 329 enum tcp_synack_type synack_type, 330 struct sk_buff *syn_skb) 331 { 332 subflow_prep_synack(sk, req, foc, synack_type); 333 334 return tcp_request_sock_ipv4_ops.send_synack(sk, dst, fl, req, foc, 335 synack_type, syn_skb); 336 } 337 338 #if IS_ENABLED(CONFIG_MPTCP_IPV6) 339 static int subflow_v6_send_synack(const struct sock *sk, struct dst_entry *dst, 340 struct flowi *fl, 341 struct request_sock *req, 342 struct tcp_fastopen_cookie *foc, 343 enum tcp_synack_type synack_type, 344 struct sk_buff *syn_skb) 345 { 346 subflow_prep_synack(sk, req, foc, synack_type); 347 348 return tcp_request_sock_ipv6_ops.send_synack(sk, dst, fl, req, foc, 349 synack_type, syn_skb); 350 } 351 352 static struct dst_entry *subflow_v6_route_req(const struct sock *sk, 353 struct sk_buff *skb, 354 struct flowi *fl, 355 struct request_sock *req) 356 { 357 struct dst_entry *dst; 358 int err; 359 360 tcp_rsk(req)->is_mptcp = 1; 361 subflow_init_req(req, sk); 362 363 dst = tcp_request_sock_ipv6_ops.route_req(sk, skb, fl, req); 364 if (!dst) 365 return NULL; 366 367 err = subflow_check_req(req, sk, skb); 368 if (err == 0) 369 return dst; 370 371 dst_release(dst); 372 if (!req->syncookie) 373 tcp6_request_sock_ops.send_reset(sk, skb); 374 return NULL; 375 } 376 #endif 377 378 /* validate received truncated hmac and create hmac for third ACK */ 379 static bool subflow_thmac_valid(struct mptcp_subflow_context *subflow) 380 { 381 u8 hmac[SHA256_DIGEST_SIZE]; 382 u64 thmac; 383 384 subflow_generate_hmac(subflow->remote_key, subflow->local_key, 385 subflow->remote_nonce, subflow->local_nonce, 386 hmac); 387 388 thmac = get_unaligned_be64(hmac); 389 pr_debug("subflow=%p, token=%u, thmac=%llu, subflow->thmac=%llu\n", 390 subflow, subflow->token, thmac, subflow->thmac); 391 392 return thmac == subflow->thmac; 393 } 394 395 void mptcp_subflow_reset(struct sock *ssk) 396 { 397 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk); 398 struct sock *sk = subflow->conn; 399 400 /* mptcp_mp_fail_no_response() can reach here on an already closed 401 * socket 402 */ 403 if (ssk->sk_state == TCP_CLOSE) 404 return; 405 406 /* must hold: tcp_done() could drop last reference on parent */ 407 sock_hold(sk); 408 409 tcp_send_active_reset(ssk, GFP_ATOMIC); 410 tcp_done(ssk); 411 if (!test_and_set_bit(MPTCP_WORK_CLOSE_SUBFLOW, &mptcp_sk(sk)->flags)) 412 mptcp_schedule_work(sk); 413 414 sock_put(sk); 415 } 416 417 static bool subflow_use_different_dport(struct mptcp_sock *msk, const struct sock *sk) 418 { 419 return inet_sk(sk)->inet_dport != inet_sk((struct sock *)msk)->inet_dport; 420 } 421 422 void __mptcp_set_connected(struct sock *sk) 423 { 424 if (sk->sk_state == TCP_SYN_SENT) { 425 inet_sk_state_store(sk, TCP_ESTABLISHED); 426 sk->sk_state_change(sk); 427 } 428 } 429 430 static void mptcp_set_connected(struct sock *sk) 431 { 432 mptcp_data_lock(sk); 433 if (!sock_owned_by_user(sk)) 434 __mptcp_set_connected(sk); 435 else 436 __set_bit(MPTCP_CONNECTED, &mptcp_sk(sk)->cb_flags); 437 mptcp_data_unlock(sk); 438 } 439 440 static void subflow_set_remote_key(struct mptcp_sock *msk, 441 struct mptcp_subflow_context *subflow, 442 const struct mptcp_options_received *mp_opt) 443 { 444 /* active MPC subflow will reach here multiple times: 445 * at subflow_finish_connect() time and at 4th ack time 446 */ 447 if (subflow->remote_key_valid) 448 return; 449 450 subflow->remote_key_valid = 1; 451 subflow->remote_key = mp_opt->sndr_key; 452 mptcp_crypto_key_sha(subflow->remote_key, NULL, &subflow->iasn); 453 subflow->iasn++; 454 455 WRITE_ONCE(msk->remote_key, subflow->remote_key); 456 WRITE_ONCE(msk->ack_seq, subflow->iasn); 457 WRITE_ONCE(msk->can_ack, true); 458 atomic64_set(&msk->rcv_wnd_sent, subflow->iasn); 459 } 460 461 static void subflow_finish_connect(struct sock *sk, const struct sk_buff *skb) 462 { 463 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(sk); 464 struct mptcp_options_received mp_opt; 465 struct sock *parent = subflow->conn; 466 struct mptcp_sock *msk; 467 468 subflow->icsk_af_ops->sk_rx_dst_set(sk, skb); 469 470 /* be sure no special action on any packet other than syn-ack */ 471 if (subflow->conn_finished) 472 return; 473 474 msk = mptcp_sk(parent); 475 mptcp_propagate_sndbuf(parent, sk); 476 subflow->rel_write_seq = 1; 477 subflow->conn_finished = 1; 478 subflow->ssn_offset = TCP_SKB_CB(skb)->seq; 479 pr_debug("subflow=%p synack seq=%x", subflow, subflow->ssn_offset); 480 481 mptcp_get_options(skb, &mp_opt); 482 if (subflow->request_mptcp) { 483 if (!(mp_opt.suboptions & OPTIONS_MPTCP_MPC)) { 484 MPTCP_INC_STATS(sock_net(sk), 485 MPTCP_MIB_MPCAPABLEACTIVEFALLBACK); 486 mptcp_do_fallback(sk); 487 pr_fallback(msk); 488 goto fallback; 489 } 490 491 if (mp_opt.suboptions & OPTION_MPTCP_CSUMREQD) 492 WRITE_ONCE(msk->csum_enabled, true); 493 if (mp_opt.deny_join_id0) 494 WRITE_ONCE(msk->pm.remote_deny_join_id0, true); 495 subflow->mp_capable = 1; 496 subflow_set_remote_key(msk, subflow, &mp_opt); 497 MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_MPCAPABLEACTIVEACK); 498 mptcp_finish_connect(sk); 499 mptcp_set_connected(parent); 500 } else if (subflow->request_join) { 501 u8 hmac[SHA256_DIGEST_SIZE]; 502 503 if (!(mp_opt.suboptions & OPTIONS_MPTCP_MPJ)) { 504 subflow->reset_reason = MPTCP_RST_EMPTCP; 505 goto do_reset; 506 } 507 508 subflow->backup = mp_opt.backup; 509 subflow->thmac = mp_opt.thmac; 510 subflow->remote_nonce = mp_opt.nonce; 511 subflow->remote_id = mp_opt.join_id; 512 pr_debug("subflow=%p, thmac=%llu, remote_nonce=%u backup=%d", 513 subflow, subflow->thmac, subflow->remote_nonce, 514 subflow->backup); 515 516 if (!subflow_thmac_valid(subflow)) { 517 MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_JOINACKMAC); 518 subflow->reset_reason = MPTCP_RST_EMPTCP; 519 goto do_reset; 520 } 521 522 if (!mptcp_finish_join(sk)) 523 goto do_reset; 524 525 subflow_generate_hmac(subflow->local_key, subflow->remote_key, 526 subflow->local_nonce, 527 subflow->remote_nonce, 528 hmac); 529 memcpy(subflow->hmac, hmac, MPTCPOPT_HMAC_LEN); 530 531 subflow->mp_join = 1; 532 MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_JOINSYNACKRX); 533 534 if (subflow_use_different_dport(msk, sk)) { 535 pr_debug("synack inet_dport=%d %d", 536 ntohs(inet_sk(sk)->inet_dport), 537 ntohs(inet_sk(parent)->inet_dport)); 538 MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_JOINPORTSYNACKRX); 539 } 540 } else if (mptcp_check_fallback(sk)) { 541 fallback: 542 mptcp_rcv_space_init(msk, sk); 543 mptcp_set_connected(parent); 544 } 545 return; 546 547 do_reset: 548 subflow->reset_transient = 0; 549 mptcp_subflow_reset(sk); 550 } 551 552 static void subflow_set_local_id(struct mptcp_subflow_context *subflow, int local_id) 553 { 554 subflow->local_id = local_id; 555 subflow->local_id_valid = 1; 556 } 557 558 static int subflow_chk_local_id(struct sock *sk) 559 { 560 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(sk); 561 struct mptcp_sock *msk = mptcp_sk(subflow->conn); 562 int err; 563 564 if (likely(subflow->local_id_valid)) 565 return 0; 566 567 err = mptcp_pm_get_local_id(msk, (struct sock_common *)sk); 568 if (err < 0) 569 return err; 570 571 subflow_set_local_id(subflow, err); 572 return 0; 573 } 574 575 static int subflow_rebuild_header(struct sock *sk) 576 { 577 int err = subflow_chk_local_id(sk); 578 579 if (unlikely(err < 0)) 580 return err; 581 582 return inet_sk_rebuild_header(sk); 583 } 584 585 #if IS_ENABLED(CONFIG_MPTCP_IPV6) 586 static int subflow_v6_rebuild_header(struct sock *sk) 587 { 588 int err = subflow_chk_local_id(sk); 589 590 if (unlikely(err < 0)) 591 return err; 592 593 return inet6_sk_rebuild_header(sk); 594 } 595 #endif 596 597 static struct request_sock_ops mptcp_subflow_v4_request_sock_ops __ro_after_init; 598 static struct tcp_request_sock_ops subflow_request_sock_ipv4_ops __ro_after_init; 599 600 static int subflow_v4_conn_request(struct sock *sk, struct sk_buff *skb) 601 { 602 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(sk); 603 604 pr_debug("subflow=%p", subflow); 605 606 /* Never answer to SYNs sent to broadcast or multicast */ 607 if (skb_rtable(skb)->rt_flags & (RTCF_BROADCAST | RTCF_MULTICAST)) 608 goto drop; 609 610 return tcp_conn_request(&mptcp_subflow_v4_request_sock_ops, 611 &subflow_request_sock_ipv4_ops, 612 sk, skb); 613 drop: 614 tcp_listendrop(sk); 615 return 0; 616 } 617 618 static void subflow_v4_req_destructor(struct request_sock *req) 619 { 620 subflow_req_destructor(req); 621 tcp_request_sock_ops.destructor(req); 622 } 623 624 #if IS_ENABLED(CONFIG_MPTCP_IPV6) 625 static struct request_sock_ops mptcp_subflow_v6_request_sock_ops __ro_after_init; 626 static struct tcp_request_sock_ops subflow_request_sock_ipv6_ops __ro_after_init; 627 static struct inet_connection_sock_af_ops subflow_v6_specific __ro_after_init; 628 static struct inet_connection_sock_af_ops subflow_v6m_specific __ro_after_init; 629 static struct proto tcpv6_prot_override __ro_after_init; 630 631 static int subflow_v6_conn_request(struct sock *sk, struct sk_buff *skb) 632 { 633 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(sk); 634 635 pr_debug("subflow=%p", subflow); 636 637 if (skb->protocol == htons(ETH_P_IP)) 638 return subflow_v4_conn_request(sk, skb); 639 640 if (!ipv6_unicast_destination(skb)) 641 goto drop; 642 643 if (ipv6_addr_v4mapped(&ipv6_hdr(skb)->saddr)) { 644 __IP6_INC_STATS(sock_net(sk), NULL, IPSTATS_MIB_INHDRERRORS); 645 return 0; 646 } 647 648 return tcp_conn_request(&mptcp_subflow_v6_request_sock_ops, 649 &subflow_request_sock_ipv6_ops, sk, skb); 650 651 drop: 652 tcp_listendrop(sk); 653 return 0; /* don't send reset */ 654 } 655 656 static void subflow_v6_req_destructor(struct request_sock *req) 657 { 658 subflow_req_destructor(req); 659 tcp6_request_sock_ops.destructor(req); 660 } 661 #endif 662 663 struct request_sock *mptcp_subflow_reqsk_alloc(const struct request_sock_ops *ops, 664 struct sock *sk_listener, 665 bool attach_listener) 666 { 667 if (ops->family == AF_INET) 668 ops = &mptcp_subflow_v4_request_sock_ops; 669 #if IS_ENABLED(CONFIG_MPTCP_IPV6) 670 else if (ops->family == AF_INET6) 671 ops = &mptcp_subflow_v6_request_sock_ops; 672 #endif 673 674 return inet_reqsk_alloc(ops, sk_listener, attach_listener); 675 } 676 EXPORT_SYMBOL(mptcp_subflow_reqsk_alloc); 677 678 /* validate hmac received in third ACK */ 679 static bool subflow_hmac_valid(const struct request_sock *req, 680 const struct mptcp_options_received *mp_opt) 681 { 682 const struct mptcp_subflow_request_sock *subflow_req; 683 u8 hmac[SHA256_DIGEST_SIZE]; 684 struct mptcp_sock *msk; 685 686 subflow_req = mptcp_subflow_rsk(req); 687 msk = subflow_req->msk; 688 if (!msk) 689 return false; 690 691 subflow_generate_hmac(msk->remote_key, msk->local_key, 692 subflow_req->remote_nonce, 693 subflow_req->local_nonce, hmac); 694 695 return !crypto_memneq(hmac, mp_opt->hmac, MPTCPOPT_HMAC_LEN); 696 } 697 698 static void mptcp_force_close(struct sock *sk) 699 { 700 /* the msk is not yet exposed to user-space, and refcount is 2 */ 701 inet_sk_state_store(sk, TCP_CLOSE); 702 sk_common_release(sk); 703 sock_put(sk); 704 } 705 706 static void subflow_ulp_fallback(struct sock *sk, 707 struct mptcp_subflow_context *old_ctx) 708 { 709 struct inet_connection_sock *icsk = inet_csk(sk); 710 711 mptcp_subflow_tcp_fallback(sk, old_ctx); 712 icsk->icsk_ulp_ops = NULL; 713 rcu_assign_pointer(icsk->icsk_ulp_data, NULL); 714 tcp_sk(sk)->is_mptcp = 0; 715 716 mptcp_subflow_ops_undo_override(sk); 717 } 718 719 void mptcp_subflow_drop_ctx(struct sock *ssk) 720 { 721 struct mptcp_subflow_context *ctx = mptcp_subflow_ctx(ssk); 722 723 if (!ctx) 724 return; 725 726 list_del(&mptcp_subflow_ctx(ssk)->node); 727 if (inet_csk(ssk)->icsk_ulp_ops) { 728 subflow_ulp_fallback(ssk, ctx); 729 if (ctx->conn) 730 sock_put(ctx->conn); 731 } 732 733 kfree_rcu(ctx, rcu); 734 } 735 736 void mptcp_subflow_fully_established(struct mptcp_subflow_context *subflow, 737 const struct mptcp_options_received *mp_opt) 738 { 739 struct mptcp_sock *msk = mptcp_sk(subflow->conn); 740 741 subflow_set_remote_key(msk, subflow, mp_opt); 742 subflow->fully_established = 1; 743 WRITE_ONCE(msk->fully_established, true); 744 745 if (subflow->is_mptfo) 746 mptcp_fastopen_gen_msk_ackseq(msk, subflow, mp_opt); 747 } 748 749 static struct sock *subflow_syn_recv_sock(const struct sock *sk, 750 struct sk_buff *skb, 751 struct request_sock *req, 752 struct dst_entry *dst, 753 struct request_sock *req_unhash, 754 bool *own_req) 755 { 756 struct mptcp_subflow_context *listener = mptcp_subflow_ctx(sk); 757 struct mptcp_subflow_request_sock *subflow_req; 758 struct mptcp_options_received mp_opt; 759 bool fallback, fallback_is_fatal; 760 struct sock *new_msk = NULL; 761 struct mptcp_sock *owner; 762 struct sock *child; 763 764 pr_debug("listener=%p, req=%p, conn=%p", listener, req, listener->conn); 765 766 /* After child creation we must look for MPC even when options 767 * are not parsed 768 */ 769 mp_opt.suboptions = 0; 770 771 /* hopefully temporary handling for MP_JOIN+syncookie */ 772 subflow_req = mptcp_subflow_rsk(req); 773 fallback_is_fatal = tcp_rsk(req)->is_mptcp && subflow_req->mp_join; 774 fallback = !tcp_rsk(req)->is_mptcp; 775 if (fallback) 776 goto create_child; 777 778 /* if the sk is MP_CAPABLE, we try to fetch the client key */ 779 if (subflow_req->mp_capable) { 780 /* we can receive and accept an in-window, out-of-order pkt, 781 * which may not carry the MP_CAPABLE opt even on mptcp enabled 782 * paths: always try to extract the peer key, and fallback 783 * for packets missing it. 784 * Even OoO DSS packets coming legitly after dropped or 785 * reordered MPC will cause fallback, but we don't have other 786 * options. 787 */ 788 mptcp_get_options(skb, &mp_opt); 789 if (!(mp_opt.suboptions & OPTIONS_MPTCP_MPC)) { 790 fallback = true; 791 goto create_child; 792 } 793 794 new_msk = mptcp_sk_clone(listener->conn, &mp_opt, req); 795 if (!new_msk) 796 fallback = true; 797 } else if (subflow_req->mp_join) { 798 mptcp_get_options(skb, &mp_opt); 799 if (!(mp_opt.suboptions & OPTIONS_MPTCP_MPJ) || 800 !subflow_hmac_valid(req, &mp_opt) || 801 !mptcp_can_accept_new_subflow(subflow_req->msk)) { 802 SUBFLOW_REQ_INC_STATS(req, MPTCP_MIB_JOINACKMAC); 803 fallback = true; 804 } 805 } 806 807 create_child: 808 child = listener->icsk_af_ops->syn_recv_sock(sk, skb, req, dst, 809 req_unhash, own_req); 810 811 if (child && *own_req) { 812 struct mptcp_subflow_context *ctx = mptcp_subflow_ctx(child); 813 814 tcp_rsk(req)->drop_req = false; 815 816 /* we need to fallback on ctx allocation failure and on pre-reqs 817 * checking above. In the latter scenario we additionally need 818 * to reset the context to non MPTCP status. 819 */ 820 if (!ctx || fallback) { 821 if (fallback_is_fatal) { 822 subflow_add_reset_reason(skb, MPTCP_RST_EMPTCP); 823 goto dispose_child; 824 } 825 826 if (new_msk) 827 mptcp_copy_inaddrs(new_msk, child); 828 mptcp_subflow_drop_ctx(child); 829 goto out; 830 } 831 832 /* ssk inherits options of listener sk */ 833 ctx->setsockopt_seq = listener->setsockopt_seq; 834 835 if (ctx->mp_capable) { 836 owner = mptcp_sk(new_msk); 837 838 /* this can't race with mptcp_close(), as the msk is 839 * not yet exposted to user-space 840 */ 841 inet_sk_state_store((void *)new_msk, TCP_ESTABLISHED); 842 843 /* record the newly created socket as the first msk 844 * subflow, but don't link it yet into conn_list 845 */ 846 WRITE_ONCE(owner->first, child); 847 848 /* new mpc subflow takes ownership of the newly 849 * created mptcp socket 850 */ 851 mptcp_sk(new_msk)->setsockopt_seq = ctx->setsockopt_seq; 852 mptcp_pm_new_connection(owner, child, 1); 853 mptcp_token_accept(subflow_req, owner); 854 ctx->conn = new_msk; 855 new_msk = NULL; 856 857 /* set msk addresses early to ensure mptcp_pm_get_local_id() 858 * uses the correct data 859 */ 860 mptcp_copy_inaddrs(ctx->conn, child); 861 mptcp_propagate_sndbuf(ctx->conn, child); 862 863 mptcp_rcv_space_init(owner, child); 864 list_add(&ctx->node, &owner->conn_list); 865 sock_hold(child); 866 867 /* with OoO packets we can reach here without ingress 868 * mpc option 869 */ 870 if (mp_opt.suboptions & OPTION_MPTCP_MPC_ACK) { 871 mptcp_subflow_fully_established(ctx, &mp_opt); 872 mptcp_pm_fully_established(owner, child, GFP_ATOMIC); 873 ctx->pm_notified = 1; 874 } 875 } else if (ctx->mp_join) { 876 owner = subflow_req->msk; 877 if (!owner) { 878 subflow_add_reset_reason(skb, MPTCP_RST_EPROHIBIT); 879 goto dispose_child; 880 } 881 882 /* move the msk reference ownership to the subflow */ 883 subflow_req->msk = NULL; 884 ctx->conn = (struct sock *)owner; 885 886 if (subflow_use_different_sport(owner, sk)) { 887 pr_debug("ack inet_sport=%d %d", 888 ntohs(inet_sk(sk)->inet_sport), 889 ntohs(inet_sk((struct sock *)owner)->inet_sport)); 890 if (!mptcp_pm_sport_in_anno_list(owner, sk)) { 891 SUBFLOW_REQ_INC_STATS(req, MPTCP_MIB_MISMATCHPORTACKRX); 892 goto dispose_child; 893 } 894 SUBFLOW_REQ_INC_STATS(req, MPTCP_MIB_JOINPORTACKRX); 895 } 896 897 if (!mptcp_finish_join(child)) 898 goto dispose_child; 899 900 SUBFLOW_REQ_INC_STATS(req, MPTCP_MIB_JOINACKRX); 901 tcp_rsk(req)->drop_req = true; 902 } 903 } 904 905 out: 906 /* dispose of the left over mptcp master, if any */ 907 if (unlikely(new_msk)) 908 mptcp_force_close(new_msk); 909 910 /* check for expected invariant - should never trigger, just help 911 * catching eariler subtle bugs 912 */ 913 WARN_ON_ONCE(child && *own_req && tcp_sk(child)->is_mptcp && 914 (!mptcp_subflow_ctx(child) || 915 !mptcp_subflow_ctx(child)->conn)); 916 return child; 917 918 dispose_child: 919 mptcp_subflow_drop_ctx(child); 920 tcp_rsk(req)->drop_req = true; 921 inet_csk_prepare_for_destroy_sock(child); 922 tcp_done(child); 923 req->rsk_ops->send_reset(sk, skb); 924 925 /* The last child reference will be released by the caller */ 926 return child; 927 } 928 929 static struct inet_connection_sock_af_ops subflow_specific __ro_after_init; 930 static struct proto tcp_prot_override __ro_after_init; 931 932 enum mapping_status { 933 MAPPING_OK, 934 MAPPING_INVALID, 935 MAPPING_EMPTY, 936 MAPPING_DATA_FIN, 937 MAPPING_DUMMY, 938 MAPPING_BAD_CSUM 939 }; 940 941 static void dbg_bad_map(struct mptcp_subflow_context *subflow, u32 ssn) 942 { 943 pr_debug("Bad mapping: ssn=%d map_seq=%d map_data_len=%d", 944 ssn, subflow->map_subflow_seq, subflow->map_data_len); 945 } 946 947 static bool skb_is_fully_mapped(struct sock *ssk, struct sk_buff *skb) 948 { 949 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk); 950 unsigned int skb_consumed; 951 952 skb_consumed = tcp_sk(ssk)->copied_seq - TCP_SKB_CB(skb)->seq; 953 if (WARN_ON_ONCE(skb_consumed >= skb->len)) 954 return true; 955 956 return skb->len - skb_consumed <= subflow->map_data_len - 957 mptcp_subflow_get_map_offset(subflow); 958 } 959 960 static bool validate_mapping(struct sock *ssk, struct sk_buff *skb) 961 { 962 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk); 963 u32 ssn = tcp_sk(ssk)->copied_seq - subflow->ssn_offset; 964 965 if (unlikely(before(ssn, subflow->map_subflow_seq))) { 966 /* Mapping covers data later in the subflow stream, 967 * currently unsupported. 968 */ 969 dbg_bad_map(subflow, ssn); 970 return false; 971 } 972 if (unlikely(!before(ssn, subflow->map_subflow_seq + 973 subflow->map_data_len))) { 974 /* Mapping does covers past subflow data, invalid */ 975 dbg_bad_map(subflow, ssn); 976 return false; 977 } 978 return true; 979 } 980 981 static enum mapping_status validate_data_csum(struct sock *ssk, struct sk_buff *skb, 982 bool csum_reqd) 983 { 984 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk); 985 u32 offset, seq, delta; 986 __sum16 csum; 987 int len; 988 989 if (!csum_reqd) 990 return MAPPING_OK; 991 992 /* mapping already validated on previous traversal */ 993 if (subflow->map_csum_len == subflow->map_data_len) 994 return MAPPING_OK; 995 996 /* traverse the receive queue, ensuring it contains a full 997 * DSS mapping and accumulating the related csum. 998 * Preserve the accoumlate csum across multiple calls, to compute 999 * the csum only once 1000 */ 1001 delta = subflow->map_data_len - subflow->map_csum_len; 1002 for (;;) { 1003 seq = tcp_sk(ssk)->copied_seq + subflow->map_csum_len; 1004 offset = seq - TCP_SKB_CB(skb)->seq; 1005 1006 /* if the current skb has not been accounted yet, csum its contents 1007 * up to the amount covered by the current DSS 1008 */ 1009 if (offset < skb->len) { 1010 __wsum csum; 1011 1012 len = min(skb->len - offset, delta); 1013 csum = skb_checksum(skb, offset, len, 0); 1014 subflow->map_data_csum = csum_block_add(subflow->map_data_csum, csum, 1015 subflow->map_csum_len); 1016 1017 delta -= len; 1018 subflow->map_csum_len += len; 1019 } 1020 if (delta == 0) 1021 break; 1022 1023 if (skb_queue_is_last(&ssk->sk_receive_queue, skb)) { 1024 /* if this subflow is closed, the partial mapping 1025 * will be never completed; flush the pending skbs, so 1026 * that subflow_sched_work_if_closed() can kick in 1027 */ 1028 if (unlikely(ssk->sk_state == TCP_CLOSE)) 1029 while ((skb = skb_peek(&ssk->sk_receive_queue))) 1030 sk_eat_skb(ssk, skb); 1031 1032 /* not enough data to validate the csum */ 1033 return MAPPING_EMPTY; 1034 } 1035 1036 /* the DSS mapping for next skbs will be validated later, 1037 * when a get_mapping_status call will process such skb 1038 */ 1039 skb = skb->next; 1040 } 1041 1042 /* note that 'map_data_len' accounts only for the carried data, does 1043 * not include the eventual seq increment due to the data fin, 1044 * while the pseudo header requires the original DSS data len, 1045 * including that 1046 */ 1047 csum = __mptcp_make_csum(subflow->map_seq, 1048 subflow->map_subflow_seq, 1049 subflow->map_data_len + subflow->map_data_fin, 1050 subflow->map_data_csum); 1051 if (unlikely(csum)) { 1052 MPTCP_INC_STATS(sock_net(ssk), MPTCP_MIB_DATACSUMERR); 1053 return MAPPING_BAD_CSUM; 1054 } 1055 1056 subflow->valid_csum_seen = 1; 1057 return MAPPING_OK; 1058 } 1059 1060 static enum mapping_status get_mapping_status(struct sock *ssk, 1061 struct mptcp_sock *msk) 1062 { 1063 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk); 1064 bool csum_reqd = READ_ONCE(msk->csum_enabled); 1065 struct mptcp_ext *mpext; 1066 struct sk_buff *skb; 1067 u16 data_len; 1068 u64 map_seq; 1069 1070 skb = skb_peek(&ssk->sk_receive_queue); 1071 if (!skb) 1072 return MAPPING_EMPTY; 1073 1074 if (mptcp_check_fallback(ssk)) 1075 return MAPPING_DUMMY; 1076 1077 mpext = mptcp_get_ext(skb); 1078 if (!mpext || !mpext->use_map) { 1079 if (!subflow->map_valid && !skb->len) { 1080 /* the TCP stack deliver 0 len FIN pkt to the receive 1081 * queue, that is the only 0len pkts ever expected here, 1082 * and we can admit no mapping only for 0 len pkts 1083 */ 1084 if (!(TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN)) 1085 WARN_ONCE(1, "0len seq %d:%d flags %x", 1086 TCP_SKB_CB(skb)->seq, 1087 TCP_SKB_CB(skb)->end_seq, 1088 TCP_SKB_CB(skb)->tcp_flags); 1089 sk_eat_skb(ssk, skb); 1090 return MAPPING_EMPTY; 1091 } 1092 1093 if (!subflow->map_valid) 1094 return MAPPING_INVALID; 1095 1096 goto validate_seq; 1097 } 1098 1099 trace_get_mapping_status(mpext); 1100 1101 data_len = mpext->data_len; 1102 if (data_len == 0) { 1103 pr_debug("infinite mapping received"); 1104 MPTCP_INC_STATS(sock_net(ssk), MPTCP_MIB_INFINITEMAPRX); 1105 subflow->map_data_len = 0; 1106 return MAPPING_INVALID; 1107 } 1108 1109 if (mpext->data_fin == 1) { 1110 if (data_len == 1) { 1111 bool updated = mptcp_update_rcv_data_fin(msk, mpext->data_seq, 1112 mpext->dsn64); 1113 pr_debug("DATA_FIN with no payload seq=%llu", mpext->data_seq); 1114 if (subflow->map_valid) { 1115 /* A DATA_FIN might arrive in a DSS 1116 * option before the previous mapping 1117 * has been fully consumed. Continue 1118 * handling the existing mapping. 1119 */ 1120 skb_ext_del(skb, SKB_EXT_MPTCP); 1121 return MAPPING_OK; 1122 } else { 1123 if (updated) 1124 mptcp_schedule_work((struct sock *)msk); 1125 1126 return MAPPING_DATA_FIN; 1127 } 1128 } else { 1129 u64 data_fin_seq = mpext->data_seq + data_len - 1; 1130 1131 /* If mpext->data_seq is a 32-bit value, data_fin_seq 1132 * must also be limited to 32 bits. 1133 */ 1134 if (!mpext->dsn64) 1135 data_fin_seq &= GENMASK_ULL(31, 0); 1136 1137 mptcp_update_rcv_data_fin(msk, data_fin_seq, mpext->dsn64); 1138 pr_debug("DATA_FIN with mapping seq=%llu dsn64=%d", 1139 data_fin_seq, mpext->dsn64); 1140 } 1141 1142 /* Adjust for DATA_FIN using 1 byte of sequence space */ 1143 data_len--; 1144 } 1145 1146 map_seq = mptcp_expand_seq(READ_ONCE(msk->ack_seq), mpext->data_seq, mpext->dsn64); 1147 WRITE_ONCE(mptcp_sk(subflow->conn)->use_64bit_ack, !!mpext->dsn64); 1148 1149 if (subflow->map_valid) { 1150 /* Allow replacing only with an identical map */ 1151 if (subflow->map_seq == map_seq && 1152 subflow->map_subflow_seq == mpext->subflow_seq && 1153 subflow->map_data_len == data_len && 1154 subflow->map_csum_reqd == mpext->csum_reqd) { 1155 skb_ext_del(skb, SKB_EXT_MPTCP); 1156 goto validate_csum; 1157 } 1158 1159 /* If this skb data are fully covered by the current mapping, 1160 * the new map would need caching, which is not supported 1161 */ 1162 if (skb_is_fully_mapped(ssk, skb)) { 1163 MPTCP_INC_STATS(sock_net(ssk), MPTCP_MIB_DSSNOMATCH); 1164 return MAPPING_INVALID; 1165 } 1166 1167 /* will validate the next map after consuming the current one */ 1168 goto validate_csum; 1169 } 1170 1171 subflow->map_seq = map_seq; 1172 subflow->map_subflow_seq = mpext->subflow_seq; 1173 subflow->map_data_len = data_len; 1174 subflow->map_valid = 1; 1175 subflow->map_data_fin = mpext->data_fin; 1176 subflow->mpc_map = mpext->mpc_map; 1177 subflow->map_csum_reqd = mpext->csum_reqd; 1178 subflow->map_csum_len = 0; 1179 subflow->map_data_csum = csum_unfold(mpext->csum); 1180 1181 /* Cfr RFC 8684 Section 3.3.0 */ 1182 if (unlikely(subflow->map_csum_reqd != csum_reqd)) 1183 return MAPPING_INVALID; 1184 1185 pr_debug("new map seq=%llu subflow_seq=%u data_len=%u csum=%d:%u", 1186 subflow->map_seq, subflow->map_subflow_seq, 1187 subflow->map_data_len, subflow->map_csum_reqd, 1188 subflow->map_data_csum); 1189 1190 validate_seq: 1191 /* we revalidate valid mapping on new skb, because we must ensure 1192 * the current skb is completely covered by the available mapping 1193 */ 1194 if (!validate_mapping(ssk, skb)) { 1195 MPTCP_INC_STATS(sock_net(ssk), MPTCP_MIB_DSSTCPMISMATCH); 1196 return MAPPING_INVALID; 1197 } 1198 1199 skb_ext_del(skb, SKB_EXT_MPTCP); 1200 1201 validate_csum: 1202 return validate_data_csum(ssk, skb, csum_reqd); 1203 } 1204 1205 static void mptcp_subflow_discard_data(struct sock *ssk, struct sk_buff *skb, 1206 u64 limit) 1207 { 1208 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk); 1209 bool fin = TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN; 1210 u32 incr; 1211 1212 incr = limit >= skb->len ? skb->len + fin : limit; 1213 1214 pr_debug("discarding=%d len=%d seq=%d", incr, skb->len, 1215 subflow->map_subflow_seq); 1216 MPTCP_INC_STATS(sock_net(ssk), MPTCP_MIB_DUPDATA); 1217 tcp_sk(ssk)->copied_seq += incr; 1218 if (!before(tcp_sk(ssk)->copied_seq, TCP_SKB_CB(skb)->end_seq)) 1219 sk_eat_skb(ssk, skb); 1220 if (mptcp_subflow_get_map_offset(subflow) >= subflow->map_data_len) 1221 subflow->map_valid = 0; 1222 } 1223 1224 /* sched mptcp worker to remove the subflow if no more data is pending */ 1225 static void subflow_sched_work_if_closed(struct mptcp_sock *msk, struct sock *ssk) 1226 { 1227 if (likely(ssk->sk_state != TCP_CLOSE)) 1228 return; 1229 1230 if (skb_queue_empty(&ssk->sk_receive_queue) && 1231 !test_and_set_bit(MPTCP_WORK_CLOSE_SUBFLOW, &msk->flags)) 1232 mptcp_schedule_work((struct sock *)msk); 1233 } 1234 1235 static bool subflow_can_fallback(struct mptcp_subflow_context *subflow) 1236 { 1237 struct mptcp_sock *msk = mptcp_sk(subflow->conn); 1238 1239 if (subflow->mp_join) 1240 return false; 1241 else if (READ_ONCE(msk->csum_enabled)) 1242 return !subflow->valid_csum_seen; 1243 else 1244 return !subflow->fully_established; 1245 } 1246 1247 static void mptcp_subflow_fail(struct mptcp_sock *msk, struct sock *ssk) 1248 { 1249 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk); 1250 unsigned long fail_tout; 1251 1252 /* greceful failure can happen only on the MPC subflow */ 1253 if (WARN_ON_ONCE(ssk != READ_ONCE(msk->first))) 1254 return; 1255 1256 /* since the close timeout take precedence on the fail one, 1257 * no need to start the latter when the first is already set 1258 */ 1259 if (sock_flag((struct sock *)msk, SOCK_DEAD)) 1260 return; 1261 1262 /* we don't need extreme accuracy here, use a zero fail_tout as special 1263 * value meaning no fail timeout at all; 1264 */ 1265 fail_tout = jiffies + TCP_RTO_MAX; 1266 if (!fail_tout) 1267 fail_tout = 1; 1268 WRITE_ONCE(subflow->fail_tout, fail_tout); 1269 tcp_send_ack(ssk); 1270 1271 mptcp_reset_timeout(msk, subflow->fail_tout); 1272 } 1273 1274 static bool subflow_check_data_avail(struct sock *ssk) 1275 { 1276 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk); 1277 enum mapping_status status; 1278 struct mptcp_sock *msk; 1279 struct sk_buff *skb; 1280 1281 if (!skb_peek(&ssk->sk_receive_queue)) 1282 WRITE_ONCE(subflow->data_avail, MPTCP_SUBFLOW_NODATA); 1283 if (subflow->data_avail) 1284 return true; 1285 1286 msk = mptcp_sk(subflow->conn); 1287 for (;;) { 1288 u64 ack_seq; 1289 u64 old_ack; 1290 1291 status = get_mapping_status(ssk, msk); 1292 trace_subflow_check_data_avail(status, skb_peek(&ssk->sk_receive_queue)); 1293 if (unlikely(status == MAPPING_INVALID || status == MAPPING_DUMMY || 1294 status == MAPPING_BAD_CSUM)) 1295 goto fallback; 1296 1297 if (status != MAPPING_OK) 1298 goto no_data; 1299 1300 skb = skb_peek(&ssk->sk_receive_queue); 1301 if (WARN_ON_ONCE(!skb)) 1302 goto no_data; 1303 1304 if (unlikely(!READ_ONCE(msk->can_ack))) 1305 goto fallback; 1306 1307 old_ack = READ_ONCE(msk->ack_seq); 1308 ack_seq = mptcp_subflow_get_mapped_dsn(subflow); 1309 pr_debug("msk ack_seq=%llx subflow ack_seq=%llx", old_ack, 1310 ack_seq); 1311 if (unlikely(before64(ack_seq, old_ack))) { 1312 mptcp_subflow_discard_data(ssk, skb, old_ack - ack_seq); 1313 continue; 1314 } 1315 1316 WRITE_ONCE(subflow->data_avail, MPTCP_SUBFLOW_DATA_AVAIL); 1317 break; 1318 } 1319 return true; 1320 1321 no_data: 1322 subflow_sched_work_if_closed(msk, ssk); 1323 return false; 1324 1325 fallback: 1326 if (!__mptcp_check_fallback(msk)) { 1327 /* RFC 8684 section 3.7. */ 1328 if (status == MAPPING_BAD_CSUM && 1329 (subflow->mp_join || subflow->valid_csum_seen)) { 1330 subflow->send_mp_fail = 1; 1331 1332 if (!READ_ONCE(msk->allow_infinite_fallback)) { 1333 subflow->reset_transient = 0; 1334 subflow->reset_reason = MPTCP_RST_EMIDDLEBOX; 1335 goto reset; 1336 } 1337 mptcp_subflow_fail(msk, ssk); 1338 WRITE_ONCE(subflow->data_avail, MPTCP_SUBFLOW_DATA_AVAIL); 1339 return true; 1340 } 1341 1342 if (!subflow_can_fallback(subflow) && subflow->map_data_len) { 1343 /* fatal protocol error, close the socket. 1344 * subflow_error_report() will introduce the appropriate barriers 1345 */ 1346 subflow->reset_transient = 0; 1347 subflow->reset_reason = MPTCP_RST_EMPTCP; 1348 1349 reset: 1350 ssk->sk_err = EBADMSG; 1351 tcp_set_state(ssk, TCP_CLOSE); 1352 while ((skb = skb_peek(&ssk->sk_receive_queue))) 1353 sk_eat_skb(ssk, skb); 1354 tcp_send_active_reset(ssk, GFP_ATOMIC); 1355 WRITE_ONCE(subflow->data_avail, MPTCP_SUBFLOW_NODATA); 1356 return false; 1357 } 1358 1359 mptcp_do_fallback(ssk); 1360 } 1361 1362 skb = skb_peek(&ssk->sk_receive_queue); 1363 subflow->map_valid = 1; 1364 subflow->map_seq = READ_ONCE(msk->ack_seq); 1365 subflow->map_data_len = skb->len; 1366 subflow->map_subflow_seq = tcp_sk(ssk)->copied_seq - subflow->ssn_offset; 1367 WRITE_ONCE(subflow->data_avail, MPTCP_SUBFLOW_DATA_AVAIL); 1368 return true; 1369 } 1370 1371 bool mptcp_subflow_data_available(struct sock *sk) 1372 { 1373 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(sk); 1374 1375 /* check if current mapping is still valid */ 1376 if (subflow->map_valid && 1377 mptcp_subflow_get_map_offset(subflow) >= subflow->map_data_len) { 1378 subflow->map_valid = 0; 1379 WRITE_ONCE(subflow->data_avail, MPTCP_SUBFLOW_NODATA); 1380 1381 pr_debug("Done with mapping: seq=%u data_len=%u", 1382 subflow->map_subflow_seq, 1383 subflow->map_data_len); 1384 } 1385 1386 return subflow_check_data_avail(sk); 1387 } 1388 1389 /* If ssk has an mptcp parent socket, use the mptcp rcvbuf occupancy, 1390 * not the ssk one. 1391 * 1392 * In mptcp, rwin is about the mptcp-level connection data. 1393 * 1394 * Data that is still on the ssk rx queue can thus be ignored, 1395 * as far as mptcp peer is concerned that data is still inflight. 1396 * DSS ACK is updated when skb is moved to the mptcp rx queue. 1397 */ 1398 void mptcp_space(const struct sock *ssk, int *space, int *full_space) 1399 { 1400 const struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk); 1401 const struct sock *sk = subflow->conn; 1402 1403 *space = __mptcp_space(sk); 1404 *full_space = tcp_full_space(sk); 1405 } 1406 1407 void __mptcp_error_report(struct sock *sk) 1408 { 1409 struct mptcp_subflow_context *subflow; 1410 struct mptcp_sock *msk = mptcp_sk(sk); 1411 1412 mptcp_for_each_subflow(msk, subflow) { 1413 struct sock *ssk = mptcp_subflow_tcp_sock(subflow); 1414 int err = sock_error(ssk); 1415 int ssk_state; 1416 1417 if (!err) 1418 continue; 1419 1420 /* only propagate errors on fallen-back sockets or 1421 * on MPC connect 1422 */ 1423 if (sk->sk_state != TCP_SYN_SENT && !__mptcp_check_fallback(msk)) 1424 continue; 1425 1426 /* We need to propagate only transition to CLOSE state. 1427 * Orphaned socket will see such state change via 1428 * subflow_sched_work_if_closed() and that path will properly 1429 * destroy the msk as needed. 1430 */ 1431 ssk_state = inet_sk_state_load(ssk); 1432 if (ssk_state == TCP_CLOSE && !sock_flag(sk, SOCK_DEAD)) 1433 inet_sk_state_store(sk, ssk_state); 1434 sk->sk_err = -err; 1435 1436 /* This barrier is coupled with smp_rmb() in mptcp_poll() */ 1437 smp_wmb(); 1438 sk_error_report(sk); 1439 break; 1440 } 1441 } 1442 1443 static void subflow_error_report(struct sock *ssk) 1444 { 1445 struct sock *sk = mptcp_subflow_ctx(ssk)->conn; 1446 1447 /* bail early if this is a no-op, so that we avoid introducing a 1448 * problematic lockdep dependency between TCP accept queue lock 1449 * and msk socket spinlock 1450 */ 1451 if (!sk->sk_socket) 1452 return; 1453 1454 mptcp_data_lock(sk); 1455 if (!sock_owned_by_user(sk)) 1456 __mptcp_error_report(sk); 1457 else 1458 __set_bit(MPTCP_ERROR_REPORT, &mptcp_sk(sk)->cb_flags); 1459 mptcp_data_unlock(sk); 1460 } 1461 1462 static void subflow_data_ready(struct sock *sk) 1463 { 1464 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(sk); 1465 u16 state = 1 << inet_sk_state_load(sk); 1466 struct sock *parent = subflow->conn; 1467 struct mptcp_sock *msk; 1468 1469 trace_sk_data_ready(sk); 1470 1471 msk = mptcp_sk(parent); 1472 if (state & TCPF_LISTEN) { 1473 /* MPJ subflow are removed from accept queue before reaching here, 1474 * avoid stray wakeups 1475 */ 1476 if (reqsk_queue_empty(&inet_csk(sk)->icsk_accept_queue)) 1477 return; 1478 1479 parent->sk_data_ready(parent); 1480 return; 1481 } 1482 1483 WARN_ON_ONCE(!__mptcp_check_fallback(msk) && !subflow->mp_capable && 1484 !subflow->mp_join && !(state & TCPF_CLOSE)); 1485 1486 if (mptcp_subflow_data_available(sk)) 1487 mptcp_data_ready(parent, sk); 1488 else if (unlikely(sk->sk_err)) 1489 subflow_error_report(sk); 1490 } 1491 1492 static void subflow_write_space(struct sock *ssk) 1493 { 1494 struct sock *sk = mptcp_subflow_ctx(ssk)->conn; 1495 1496 mptcp_propagate_sndbuf(sk, ssk); 1497 mptcp_write_space(sk); 1498 } 1499 1500 static const struct inet_connection_sock_af_ops * 1501 subflow_default_af_ops(struct sock *sk) 1502 { 1503 #if IS_ENABLED(CONFIG_MPTCP_IPV6) 1504 if (sk->sk_family == AF_INET6) 1505 return &subflow_v6_specific; 1506 #endif 1507 return &subflow_specific; 1508 } 1509 1510 #if IS_ENABLED(CONFIG_MPTCP_IPV6) 1511 void mptcpv6_handle_mapped(struct sock *sk, bool mapped) 1512 { 1513 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(sk); 1514 struct inet_connection_sock *icsk = inet_csk(sk); 1515 const struct inet_connection_sock_af_ops *target; 1516 1517 target = mapped ? &subflow_v6m_specific : subflow_default_af_ops(sk); 1518 1519 pr_debug("subflow=%p family=%d ops=%p target=%p mapped=%d", 1520 subflow, sk->sk_family, icsk->icsk_af_ops, target, mapped); 1521 1522 if (likely(icsk->icsk_af_ops == target)) 1523 return; 1524 1525 subflow->icsk_af_ops = icsk->icsk_af_ops; 1526 icsk->icsk_af_ops = target; 1527 } 1528 #endif 1529 1530 void mptcp_info2sockaddr(const struct mptcp_addr_info *info, 1531 struct sockaddr_storage *addr, 1532 unsigned short family) 1533 { 1534 memset(addr, 0, sizeof(*addr)); 1535 addr->ss_family = family; 1536 if (addr->ss_family == AF_INET) { 1537 struct sockaddr_in *in_addr = (struct sockaddr_in *)addr; 1538 1539 if (info->family == AF_INET) 1540 in_addr->sin_addr = info->addr; 1541 #if IS_ENABLED(CONFIG_MPTCP_IPV6) 1542 else if (ipv6_addr_v4mapped(&info->addr6)) 1543 in_addr->sin_addr.s_addr = info->addr6.s6_addr32[3]; 1544 #endif 1545 in_addr->sin_port = info->port; 1546 } 1547 #if IS_ENABLED(CONFIG_MPTCP_IPV6) 1548 else if (addr->ss_family == AF_INET6) { 1549 struct sockaddr_in6 *in6_addr = (struct sockaddr_in6 *)addr; 1550 1551 if (info->family == AF_INET) 1552 ipv6_addr_set_v4mapped(info->addr.s_addr, 1553 &in6_addr->sin6_addr); 1554 else 1555 in6_addr->sin6_addr = info->addr6; 1556 in6_addr->sin6_port = info->port; 1557 } 1558 #endif 1559 } 1560 1561 int __mptcp_subflow_connect(struct sock *sk, const struct mptcp_addr_info *loc, 1562 const struct mptcp_addr_info *remote) 1563 { 1564 struct mptcp_sock *msk = mptcp_sk(sk); 1565 struct mptcp_subflow_context *subflow; 1566 struct sockaddr_storage addr; 1567 int remote_id = remote->id; 1568 int local_id = loc->id; 1569 int err = -ENOTCONN; 1570 struct socket *sf; 1571 struct sock *ssk; 1572 u32 remote_token; 1573 int addrlen; 1574 int ifindex; 1575 u8 flags; 1576 1577 if (!mptcp_is_fully_established(sk)) 1578 goto err_out; 1579 1580 err = mptcp_subflow_create_socket(sk, loc->family, &sf); 1581 if (err) 1582 goto err_out; 1583 1584 ssk = sf->sk; 1585 subflow = mptcp_subflow_ctx(ssk); 1586 do { 1587 get_random_bytes(&subflow->local_nonce, sizeof(u32)); 1588 } while (!subflow->local_nonce); 1589 1590 if (local_id) 1591 subflow_set_local_id(subflow, local_id); 1592 1593 mptcp_pm_get_flags_and_ifindex_by_id(msk, local_id, 1594 &flags, &ifindex); 1595 subflow->remote_key_valid = 1; 1596 subflow->remote_key = msk->remote_key; 1597 subflow->local_key = msk->local_key; 1598 subflow->token = msk->token; 1599 mptcp_info2sockaddr(loc, &addr, ssk->sk_family); 1600 1601 addrlen = sizeof(struct sockaddr_in); 1602 #if IS_ENABLED(CONFIG_MPTCP_IPV6) 1603 if (addr.ss_family == AF_INET6) 1604 addrlen = sizeof(struct sockaddr_in6); 1605 #endif 1606 mptcp_sockopt_sync(msk, ssk); 1607 1608 ssk->sk_bound_dev_if = ifindex; 1609 err = kernel_bind(sf, (struct sockaddr *)&addr, addrlen); 1610 if (err) 1611 goto failed; 1612 1613 mptcp_crypto_key_sha(subflow->remote_key, &remote_token, NULL); 1614 pr_debug("msk=%p remote_token=%u local_id=%d remote_id=%d", msk, 1615 remote_token, local_id, remote_id); 1616 subflow->remote_token = remote_token; 1617 subflow->remote_id = remote_id; 1618 subflow->request_join = 1; 1619 subflow->request_bkup = !!(flags & MPTCP_PM_ADDR_FLAG_BACKUP); 1620 mptcp_info2sockaddr(remote, &addr, ssk->sk_family); 1621 1622 sock_hold(ssk); 1623 list_add_tail(&subflow->node, &msk->conn_list); 1624 err = kernel_connect(sf, (struct sockaddr *)&addr, addrlen, O_NONBLOCK); 1625 if (err && err != -EINPROGRESS) 1626 goto failed_unlink; 1627 1628 /* discard the subflow socket */ 1629 mptcp_sock_graft(ssk, sk->sk_socket); 1630 iput(SOCK_INODE(sf)); 1631 WRITE_ONCE(msk->allow_infinite_fallback, false); 1632 return 0; 1633 1634 failed_unlink: 1635 list_del(&subflow->node); 1636 sock_put(mptcp_subflow_tcp_sock(subflow)); 1637 1638 failed: 1639 subflow->disposable = 1; 1640 sock_release(sf); 1641 1642 err_out: 1643 /* we account subflows before the creation, and this failures will not 1644 * be caught by sk_state_change() 1645 */ 1646 mptcp_pm_close_subflow(msk); 1647 return err; 1648 } 1649 1650 static void mptcp_attach_cgroup(struct sock *parent, struct sock *child) 1651 { 1652 #ifdef CONFIG_SOCK_CGROUP_DATA 1653 struct sock_cgroup_data *parent_skcd = &parent->sk_cgrp_data, 1654 *child_skcd = &child->sk_cgrp_data; 1655 1656 /* only the additional subflows created by kworkers have to be modified */ 1657 if (cgroup_id(sock_cgroup_ptr(parent_skcd)) != 1658 cgroup_id(sock_cgroup_ptr(child_skcd))) { 1659 #ifdef CONFIG_MEMCG 1660 struct mem_cgroup *memcg = parent->sk_memcg; 1661 1662 mem_cgroup_sk_free(child); 1663 if (memcg && css_tryget(&memcg->css)) 1664 child->sk_memcg = memcg; 1665 #endif /* CONFIG_MEMCG */ 1666 1667 cgroup_sk_free(child_skcd); 1668 *child_skcd = *parent_skcd; 1669 cgroup_sk_clone(child_skcd); 1670 } 1671 #endif /* CONFIG_SOCK_CGROUP_DATA */ 1672 } 1673 1674 static void mptcp_subflow_ops_override(struct sock *ssk) 1675 { 1676 #if IS_ENABLED(CONFIG_MPTCP_IPV6) 1677 if (ssk->sk_prot == &tcpv6_prot) 1678 ssk->sk_prot = &tcpv6_prot_override; 1679 else 1680 #endif 1681 ssk->sk_prot = &tcp_prot_override; 1682 } 1683 1684 static void mptcp_subflow_ops_undo_override(struct sock *ssk) 1685 { 1686 #if IS_ENABLED(CONFIG_MPTCP_IPV6) 1687 if (ssk->sk_prot == &tcpv6_prot_override) 1688 ssk->sk_prot = &tcpv6_prot; 1689 else 1690 #endif 1691 ssk->sk_prot = &tcp_prot; 1692 } 1693 1694 int mptcp_subflow_create_socket(struct sock *sk, unsigned short family, 1695 struct socket **new_sock) 1696 { 1697 struct mptcp_subflow_context *subflow; 1698 struct net *net = sock_net(sk); 1699 struct socket *sf; 1700 int err; 1701 1702 /* un-accepted server sockets can reach here - on bad configuration 1703 * bail early to avoid greater trouble later 1704 */ 1705 if (unlikely(!sk->sk_socket)) 1706 return -EINVAL; 1707 1708 err = sock_create_kern(net, family, SOCK_STREAM, IPPROTO_TCP, &sf); 1709 if (err) 1710 return err; 1711 1712 lock_sock_nested(sf->sk, SINGLE_DEPTH_NESTING); 1713 1714 /* the newly created socket has to be in the same cgroup as its parent */ 1715 mptcp_attach_cgroup(sk, sf->sk); 1716 1717 /* kernel sockets do not by default acquire net ref, but TCP timer 1718 * needs it. 1719 * Update ns_tracker to current stack trace and refcounted tracker. 1720 */ 1721 __netns_tracker_free(net, &sf->sk->ns_tracker, false); 1722 sf->sk->sk_net_refcnt = 1; 1723 get_net_track(net, &sf->sk->ns_tracker, GFP_KERNEL); 1724 sock_inuse_add(net, 1); 1725 err = tcp_set_ulp(sf->sk, "mptcp"); 1726 release_sock(sf->sk); 1727 1728 if (err) { 1729 sock_release(sf); 1730 return err; 1731 } 1732 1733 /* the newly created socket really belongs to the owning MPTCP master 1734 * socket, even if for additional subflows the allocation is performed 1735 * by a kernel workqueue. Adjust inode references, so that the 1736 * procfs/diag interfaces really show this one belonging to the correct 1737 * user. 1738 */ 1739 SOCK_INODE(sf)->i_ino = SOCK_INODE(sk->sk_socket)->i_ino; 1740 SOCK_INODE(sf)->i_uid = SOCK_INODE(sk->sk_socket)->i_uid; 1741 SOCK_INODE(sf)->i_gid = SOCK_INODE(sk->sk_socket)->i_gid; 1742 1743 subflow = mptcp_subflow_ctx(sf->sk); 1744 pr_debug("subflow=%p", subflow); 1745 1746 *new_sock = sf; 1747 sock_hold(sk); 1748 subflow->conn = sk; 1749 mptcp_subflow_ops_override(sf->sk); 1750 1751 return 0; 1752 } 1753 1754 static struct mptcp_subflow_context *subflow_create_ctx(struct sock *sk, 1755 gfp_t priority) 1756 { 1757 struct inet_connection_sock *icsk = inet_csk(sk); 1758 struct mptcp_subflow_context *ctx; 1759 1760 ctx = kzalloc(sizeof(*ctx), priority); 1761 if (!ctx) 1762 return NULL; 1763 1764 rcu_assign_pointer(icsk->icsk_ulp_data, ctx); 1765 INIT_LIST_HEAD(&ctx->node); 1766 INIT_LIST_HEAD(&ctx->delegated_node); 1767 1768 pr_debug("subflow=%p", ctx); 1769 1770 ctx->tcp_sock = sk; 1771 1772 return ctx; 1773 } 1774 1775 static void __subflow_state_change(struct sock *sk) 1776 { 1777 struct socket_wq *wq; 1778 1779 rcu_read_lock(); 1780 wq = rcu_dereference(sk->sk_wq); 1781 if (skwq_has_sleeper(wq)) 1782 wake_up_interruptible_all(&wq->wait); 1783 rcu_read_unlock(); 1784 } 1785 1786 static bool subflow_is_done(const struct sock *sk) 1787 { 1788 return sk->sk_shutdown & RCV_SHUTDOWN || sk->sk_state == TCP_CLOSE; 1789 } 1790 1791 static void subflow_state_change(struct sock *sk) 1792 { 1793 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(sk); 1794 struct sock *parent = subflow->conn; 1795 1796 __subflow_state_change(sk); 1797 1798 if (subflow_simultaneous_connect(sk)) { 1799 mptcp_propagate_sndbuf(parent, sk); 1800 mptcp_do_fallback(sk); 1801 mptcp_rcv_space_init(mptcp_sk(parent), sk); 1802 pr_fallback(mptcp_sk(parent)); 1803 subflow->conn_finished = 1; 1804 mptcp_set_connected(parent); 1805 } 1806 1807 /* as recvmsg() does not acquire the subflow socket for ssk selection 1808 * a fin packet carrying a DSS can be unnoticed if we don't trigger 1809 * the data available machinery here. 1810 */ 1811 if (mptcp_subflow_data_available(sk)) 1812 mptcp_data_ready(parent, sk); 1813 else if (unlikely(sk->sk_err)) 1814 subflow_error_report(sk); 1815 1816 subflow_sched_work_if_closed(mptcp_sk(parent), sk); 1817 1818 if (__mptcp_check_fallback(mptcp_sk(parent)) && 1819 !subflow->rx_eof && subflow_is_done(sk)) { 1820 subflow->rx_eof = 1; 1821 mptcp_subflow_eof(parent); 1822 } 1823 } 1824 1825 void mptcp_subflow_queue_clean(struct sock *listener_sk, struct sock *listener_ssk) 1826 { 1827 struct request_sock_queue *queue = &inet_csk(listener_ssk)->icsk_accept_queue; 1828 struct mptcp_sock *msk, *next, *head = NULL; 1829 struct request_sock *req; 1830 struct sock *sk; 1831 1832 /* build a list of all unaccepted mptcp sockets */ 1833 spin_lock_bh(&queue->rskq_lock); 1834 for (req = queue->rskq_accept_head; req; req = req->dl_next) { 1835 struct mptcp_subflow_context *subflow; 1836 struct sock *ssk = req->sk; 1837 1838 if (!sk_is_mptcp(ssk)) 1839 continue; 1840 1841 subflow = mptcp_subflow_ctx(ssk); 1842 if (!subflow || !subflow->conn) 1843 continue; 1844 1845 /* skip if already in list */ 1846 sk = subflow->conn; 1847 msk = mptcp_sk(sk); 1848 if (msk->dl_next || msk == head) 1849 continue; 1850 1851 sock_hold(sk); 1852 msk->dl_next = head; 1853 head = msk; 1854 } 1855 spin_unlock_bh(&queue->rskq_lock); 1856 if (!head) 1857 return; 1858 1859 /* can't acquire the msk socket lock under the subflow one, 1860 * or will cause ABBA deadlock 1861 */ 1862 release_sock(listener_ssk); 1863 1864 for (msk = head; msk; msk = next) { 1865 sk = (struct sock *)msk; 1866 1867 lock_sock_nested(sk, SINGLE_DEPTH_NESTING); 1868 next = msk->dl_next; 1869 msk->dl_next = NULL; 1870 1871 __mptcp_unaccepted_force_close(sk); 1872 release_sock(sk); 1873 1874 /* lockdep will report a false positive ABBA deadlock 1875 * between cancel_work_sync and the listener socket. 1876 * The involved locks belong to different sockets WRT 1877 * the existing AB chain. 1878 * Using a per socket key is problematic as key 1879 * deregistration requires process context and must be 1880 * performed at socket disposal time, in atomic 1881 * context. 1882 * Just tell lockdep to consider the listener socket 1883 * released here. 1884 */ 1885 mutex_release(&listener_sk->sk_lock.dep_map, _RET_IP_); 1886 mptcp_cancel_work(sk); 1887 mutex_acquire(&listener_sk->sk_lock.dep_map, 0, 0, _RET_IP_); 1888 1889 sock_put(sk); 1890 } 1891 1892 /* we are still under the listener msk socket lock */ 1893 lock_sock_nested(listener_ssk, SINGLE_DEPTH_NESTING); 1894 } 1895 1896 static int subflow_ulp_init(struct sock *sk) 1897 { 1898 struct inet_connection_sock *icsk = inet_csk(sk); 1899 struct mptcp_subflow_context *ctx; 1900 struct tcp_sock *tp = tcp_sk(sk); 1901 int err = 0; 1902 1903 /* disallow attaching ULP to a socket unless it has been 1904 * created with sock_create_kern() 1905 */ 1906 if (!sk->sk_kern_sock) { 1907 err = -EOPNOTSUPP; 1908 goto out; 1909 } 1910 1911 ctx = subflow_create_ctx(sk, GFP_KERNEL); 1912 if (!ctx) { 1913 err = -ENOMEM; 1914 goto out; 1915 } 1916 1917 pr_debug("subflow=%p, family=%d", ctx, sk->sk_family); 1918 1919 tp->is_mptcp = 1; 1920 ctx->icsk_af_ops = icsk->icsk_af_ops; 1921 icsk->icsk_af_ops = subflow_default_af_ops(sk); 1922 ctx->tcp_state_change = sk->sk_state_change; 1923 ctx->tcp_error_report = sk->sk_error_report; 1924 1925 WARN_ON_ONCE(sk->sk_data_ready != sock_def_readable); 1926 WARN_ON_ONCE(sk->sk_write_space != sk_stream_write_space); 1927 1928 sk->sk_data_ready = subflow_data_ready; 1929 sk->sk_write_space = subflow_write_space; 1930 sk->sk_state_change = subflow_state_change; 1931 sk->sk_error_report = subflow_error_report; 1932 out: 1933 return err; 1934 } 1935 1936 static void subflow_ulp_release(struct sock *ssk) 1937 { 1938 struct mptcp_subflow_context *ctx = mptcp_subflow_ctx(ssk); 1939 bool release = true; 1940 struct sock *sk; 1941 1942 if (!ctx) 1943 return; 1944 1945 sk = ctx->conn; 1946 if (sk) { 1947 /* if the msk has been orphaned, keep the ctx 1948 * alive, will be freed by __mptcp_close_ssk(), 1949 * when the subflow is still unaccepted 1950 */ 1951 release = ctx->disposable || list_empty(&ctx->node); 1952 1953 /* inet_child_forget() does not call sk_state_change(), 1954 * explicitly trigger the socket close machinery 1955 */ 1956 if (!release && !test_and_set_bit(MPTCP_WORK_CLOSE_SUBFLOW, 1957 &mptcp_sk(sk)->flags)) 1958 mptcp_schedule_work(sk); 1959 sock_put(sk); 1960 } 1961 1962 mptcp_subflow_ops_undo_override(ssk); 1963 if (release) 1964 kfree_rcu(ctx, rcu); 1965 } 1966 1967 static void subflow_ulp_clone(const struct request_sock *req, 1968 struct sock *newsk, 1969 const gfp_t priority) 1970 { 1971 struct mptcp_subflow_request_sock *subflow_req = mptcp_subflow_rsk(req); 1972 struct mptcp_subflow_context *old_ctx = mptcp_subflow_ctx(newsk); 1973 struct mptcp_subflow_context *new_ctx; 1974 1975 if (!tcp_rsk(req)->is_mptcp || 1976 (!subflow_req->mp_capable && !subflow_req->mp_join)) { 1977 subflow_ulp_fallback(newsk, old_ctx); 1978 return; 1979 } 1980 1981 new_ctx = subflow_create_ctx(newsk, priority); 1982 if (!new_ctx) { 1983 subflow_ulp_fallback(newsk, old_ctx); 1984 return; 1985 } 1986 1987 new_ctx->conn_finished = 1; 1988 new_ctx->icsk_af_ops = old_ctx->icsk_af_ops; 1989 new_ctx->tcp_state_change = old_ctx->tcp_state_change; 1990 new_ctx->tcp_error_report = old_ctx->tcp_error_report; 1991 new_ctx->rel_write_seq = 1; 1992 new_ctx->tcp_sock = newsk; 1993 1994 if (subflow_req->mp_capable) { 1995 /* see comments in subflow_syn_recv_sock(), MPTCP connection 1996 * is fully established only after we receive the remote key 1997 */ 1998 new_ctx->mp_capable = 1; 1999 new_ctx->local_key = subflow_req->local_key; 2000 new_ctx->token = subflow_req->token; 2001 new_ctx->ssn_offset = subflow_req->ssn_offset; 2002 new_ctx->idsn = subflow_req->idsn; 2003 2004 /* this is the first subflow, id is always 0 */ 2005 new_ctx->local_id_valid = 1; 2006 } else if (subflow_req->mp_join) { 2007 new_ctx->ssn_offset = subflow_req->ssn_offset; 2008 new_ctx->mp_join = 1; 2009 new_ctx->fully_established = 1; 2010 new_ctx->remote_key_valid = 1; 2011 new_ctx->backup = subflow_req->backup; 2012 new_ctx->remote_id = subflow_req->remote_id; 2013 new_ctx->token = subflow_req->token; 2014 new_ctx->thmac = subflow_req->thmac; 2015 2016 /* the subflow req id is valid, fetched via subflow_check_req() 2017 * and subflow_token_join_request() 2018 */ 2019 subflow_set_local_id(new_ctx, subflow_req->local_id); 2020 } 2021 } 2022 2023 static void tcp_release_cb_override(struct sock *ssk) 2024 { 2025 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk); 2026 2027 if (mptcp_subflow_has_delegated_action(subflow)) 2028 mptcp_subflow_process_delegated(ssk); 2029 2030 tcp_release_cb(ssk); 2031 } 2032 2033 static struct tcp_ulp_ops subflow_ulp_ops __read_mostly = { 2034 .name = "mptcp", 2035 .owner = THIS_MODULE, 2036 .init = subflow_ulp_init, 2037 .release = subflow_ulp_release, 2038 .clone = subflow_ulp_clone, 2039 }; 2040 2041 static int subflow_ops_init(struct request_sock_ops *subflow_ops) 2042 { 2043 subflow_ops->obj_size = sizeof(struct mptcp_subflow_request_sock); 2044 2045 subflow_ops->slab = kmem_cache_create(subflow_ops->slab_name, 2046 subflow_ops->obj_size, 0, 2047 SLAB_ACCOUNT | 2048 SLAB_TYPESAFE_BY_RCU, 2049 NULL); 2050 if (!subflow_ops->slab) 2051 return -ENOMEM; 2052 2053 return 0; 2054 } 2055 2056 void __init mptcp_subflow_init(void) 2057 { 2058 mptcp_subflow_v4_request_sock_ops = tcp_request_sock_ops; 2059 mptcp_subflow_v4_request_sock_ops.slab_name = "request_sock_subflow_v4"; 2060 mptcp_subflow_v4_request_sock_ops.destructor = subflow_v4_req_destructor; 2061 2062 if (subflow_ops_init(&mptcp_subflow_v4_request_sock_ops) != 0) 2063 panic("MPTCP: failed to init subflow v4 request sock ops\n"); 2064 2065 subflow_request_sock_ipv4_ops = tcp_request_sock_ipv4_ops; 2066 subflow_request_sock_ipv4_ops.route_req = subflow_v4_route_req; 2067 subflow_request_sock_ipv4_ops.send_synack = subflow_v4_send_synack; 2068 2069 subflow_specific = ipv4_specific; 2070 subflow_specific.conn_request = subflow_v4_conn_request; 2071 subflow_specific.syn_recv_sock = subflow_syn_recv_sock; 2072 subflow_specific.sk_rx_dst_set = subflow_finish_connect; 2073 subflow_specific.rebuild_header = subflow_rebuild_header; 2074 2075 tcp_prot_override = tcp_prot; 2076 tcp_prot_override.release_cb = tcp_release_cb_override; 2077 2078 #if IS_ENABLED(CONFIG_MPTCP_IPV6) 2079 /* In struct mptcp_subflow_request_sock, we assume the TCP request sock 2080 * structures for v4 and v6 have the same size. It should not changed in 2081 * the future but better to make sure to be warned if it is no longer 2082 * the case. 2083 */ 2084 BUILD_BUG_ON(sizeof(struct tcp_request_sock) != sizeof(struct tcp6_request_sock)); 2085 2086 mptcp_subflow_v6_request_sock_ops = tcp6_request_sock_ops; 2087 mptcp_subflow_v6_request_sock_ops.slab_name = "request_sock_subflow_v6"; 2088 mptcp_subflow_v6_request_sock_ops.destructor = subflow_v6_req_destructor; 2089 2090 if (subflow_ops_init(&mptcp_subflow_v6_request_sock_ops) != 0) 2091 panic("MPTCP: failed to init subflow v6 request sock ops\n"); 2092 2093 subflow_request_sock_ipv6_ops = tcp_request_sock_ipv6_ops; 2094 subflow_request_sock_ipv6_ops.route_req = subflow_v6_route_req; 2095 subflow_request_sock_ipv6_ops.send_synack = subflow_v6_send_synack; 2096 2097 subflow_v6_specific = ipv6_specific; 2098 subflow_v6_specific.conn_request = subflow_v6_conn_request; 2099 subflow_v6_specific.syn_recv_sock = subflow_syn_recv_sock; 2100 subflow_v6_specific.sk_rx_dst_set = subflow_finish_connect; 2101 subflow_v6_specific.rebuild_header = subflow_v6_rebuild_header; 2102 2103 subflow_v6m_specific = subflow_v6_specific; 2104 subflow_v6m_specific.queue_xmit = ipv4_specific.queue_xmit; 2105 subflow_v6m_specific.send_check = ipv4_specific.send_check; 2106 subflow_v6m_specific.net_header_len = ipv4_specific.net_header_len; 2107 subflow_v6m_specific.mtu_reduced = ipv4_specific.mtu_reduced; 2108 subflow_v6m_specific.net_frag_header_len = 0; 2109 subflow_v6m_specific.rebuild_header = subflow_rebuild_header; 2110 2111 tcpv6_prot_override = tcpv6_prot; 2112 tcpv6_prot_override.release_cb = tcp_release_cb_override; 2113 #endif 2114 2115 mptcp_diag_subflow_init(&subflow_ulp_ops); 2116 2117 if (tcp_register_ulp(&subflow_ulp_ops) != 0) 2118 panic("MPTCP: failed to register subflows to ULP\n"); 2119 } 2120