1 // SPDX-License-Identifier: GPL-2.0 2 /* Multipath TCP 3 * 4 * Copyright (c) 2017 - 2019, Intel Corporation. 5 */ 6 7 #define pr_fmt(fmt) "MPTCP: " fmt 8 9 #include <linux/kernel.h> 10 #include <linux/module.h> 11 #include <linux/netdevice.h> 12 #include <crypto/algapi.h> 13 #include <crypto/sha2.h> 14 #include <net/sock.h> 15 #include <net/inet_common.h> 16 #include <net/inet_hashtables.h> 17 #include <net/protocol.h> 18 #include <net/tcp.h> 19 #if IS_ENABLED(CONFIG_MPTCP_IPV6) 20 #include <net/ip6_route.h> 21 #include <net/transp_v6.h> 22 #endif 23 #include <net/mptcp.h> 24 #include <uapi/linux/mptcp.h> 25 #include "protocol.h" 26 #include "mib.h" 27 28 #include <trace/events/mptcp.h> 29 #include <trace/events/sock.h> 30 31 static void mptcp_subflow_ops_undo_override(struct sock *ssk); 32 33 static void SUBFLOW_REQ_INC_STATS(struct request_sock *req, 34 enum linux_mptcp_mib_field field) 35 { 36 MPTCP_INC_STATS(sock_net(req_to_sk(req)), field); 37 } 38 39 static void subflow_req_destructor(struct request_sock *req) 40 { 41 struct mptcp_subflow_request_sock *subflow_req = mptcp_subflow_rsk(req); 42 43 pr_debug("subflow_req=%p", subflow_req); 44 45 if (subflow_req->msk) 46 sock_put((struct sock *)subflow_req->msk); 47 48 mptcp_token_destroy_request(req); 49 } 50 51 static void subflow_generate_hmac(u64 key1, u64 key2, u32 nonce1, u32 nonce2, 52 void *hmac) 53 { 54 u8 msg[8]; 55 56 put_unaligned_be32(nonce1, &msg[0]); 57 put_unaligned_be32(nonce2, &msg[4]); 58 59 mptcp_crypto_hmac_sha(key1, key2, msg, 8, hmac); 60 } 61 62 static bool mptcp_can_accept_new_subflow(const struct mptcp_sock *msk) 63 { 64 return mptcp_is_fully_established((void *)msk) && 65 ((mptcp_pm_is_userspace(msk) && 66 mptcp_userspace_pm_active(msk)) || 67 READ_ONCE(msk->pm.accept_subflow)); 68 } 69 70 /* validate received token and create truncated hmac and nonce for SYN-ACK */ 71 static void subflow_req_create_thmac(struct mptcp_subflow_request_sock *subflow_req) 72 { 73 struct mptcp_sock *msk = subflow_req->msk; 74 u8 hmac[SHA256_DIGEST_SIZE]; 75 76 get_random_bytes(&subflow_req->local_nonce, sizeof(u32)); 77 78 subflow_generate_hmac(msk->local_key, msk->remote_key, 79 subflow_req->local_nonce, 80 subflow_req->remote_nonce, hmac); 81 82 subflow_req->thmac = get_unaligned_be64(hmac); 83 } 84 85 static struct mptcp_sock *subflow_token_join_request(struct request_sock *req) 86 { 87 struct mptcp_subflow_request_sock *subflow_req = mptcp_subflow_rsk(req); 88 struct mptcp_sock *msk; 89 int local_id; 90 91 msk = mptcp_token_get_sock(sock_net(req_to_sk(req)), subflow_req->token); 92 if (!msk) { 93 SUBFLOW_REQ_INC_STATS(req, MPTCP_MIB_JOINNOTOKEN); 94 return NULL; 95 } 96 97 local_id = mptcp_pm_get_local_id(msk, (struct sock_common *)req); 98 if (local_id < 0) { 99 sock_put((struct sock *)msk); 100 return NULL; 101 } 102 subflow_req->local_id = local_id; 103 104 return msk; 105 } 106 107 static void subflow_init_req(struct request_sock *req, const struct sock *sk_listener) 108 { 109 struct mptcp_subflow_request_sock *subflow_req = mptcp_subflow_rsk(req); 110 111 subflow_req->mp_capable = 0; 112 subflow_req->mp_join = 0; 113 subflow_req->csum_reqd = mptcp_is_checksum_enabled(sock_net(sk_listener)); 114 subflow_req->allow_join_id0 = mptcp_allow_join_id0(sock_net(sk_listener)); 115 subflow_req->msk = NULL; 116 mptcp_token_init_request(req); 117 } 118 119 static bool subflow_use_different_sport(struct mptcp_sock *msk, const struct sock *sk) 120 { 121 return inet_sk(sk)->inet_sport != inet_sk((struct sock *)msk)->inet_sport; 122 } 123 124 static void subflow_add_reset_reason(struct sk_buff *skb, u8 reason) 125 { 126 struct mptcp_ext *mpext = skb_ext_add(skb, SKB_EXT_MPTCP); 127 128 if (mpext) { 129 memset(mpext, 0, sizeof(*mpext)); 130 mpext->reset_reason = reason; 131 } 132 } 133 134 /* Init mptcp request socket. 135 * 136 * Returns an error code if a JOIN has failed and a TCP reset 137 * should be sent. 138 */ 139 static int subflow_check_req(struct request_sock *req, 140 const struct sock *sk_listener, 141 struct sk_buff *skb) 142 { 143 struct mptcp_subflow_context *listener = mptcp_subflow_ctx(sk_listener); 144 struct mptcp_subflow_request_sock *subflow_req = mptcp_subflow_rsk(req); 145 struct mptcp_options_received mp_opt; 146 bool opt_mp_capable, opt_mp_join; 147 148 pr_debug("subflow_req=%p, listener=%p", subflow_req, listener); 149 150 #ifdef CONFIG_TCP_MD5SIG 151 /* no MPTCP if MD5SIG is enabled on this socket or we may run out of 152 * TCP option space. 153 */ 154 if (rcu_access_pointer(tcp_sk(sk_listener)->md5sig_info)) 155 return -EINVAL; 156 #endif 157 158 mptcp_get_options(skb, &mp_opt); 159 160 opt_mp_capable = !!(mp_opt.suboptions & OPTION_MPTCP_MPC_SYN); 161 opt_mp_join = !!(mp_opt.suboptions & OPTION_MPTCP_MPJ_SYN); 162 if (opt_mp_capable) { 163 SUBFLOW_REQ_INC_STATS(req, MPTCP_MIB_MPCAPABLEPASSIVE); 164 165 if (opt_mp_join) 166 return 0; 167 } else if (opt_mp_join) { 168 SUBFLOW_REQ_INC_STATS(req, MPTCP_MIB_JOINSYNRX); 169 170 if (mp_opt.backup) 171 SUBFLOW_REQ_INC_STATS(req, MPTCP_MIB_JOINSYNBACKUPRX); 172 } 173 174 if (opt_mp_capable && listener->request_mptcp) { 175 int err, retries = MPTCP_TOKEN_MAX_RETRIES; 176 177 subflow_req->ssn_offset = TCP_SKB_CB(skb)->seq; 178 again: 179 do { 180 get_random_bytes(&subflow_req->local_key, sizeof(subflow_req->local_key)); 181 } while (subflow_req->local_key == 0); 182 183 if (unlikely(req->syncookie)) { 184 mptcp_crypto_key_sha(subflow_req->local_key, 185 &subflow_req->token, 186 &subflow_req->idsn); 187 if (mptcp_token_exists(subflow_req->token)) { 188 if (retries-- > 0) 189 goto again; 190 SUBFLOW_REQ_INC_STATS(req, MPTCP_MIB_TOKENFALLBACKINIT); 191 } else { 192 subflow_req->mp_capable = 1; 193 } 194 return 0; 195 } 196 197 err = mptcp_token_new_request(req); 198 if (err == 0) 199 subflow_req->mp_capable = 1; 200 else if (retries-- > 0) 201 goto again; 202 else 203 SUBFLOW_REQ_INC_STATS(req, MPTCP_MIB_TOKENFALLBACKINIT); 204 205 } else if (opt_mp_join && listener->request_mptcp) { 206 subflow_req->ssn_offset = TCP_SKB_CB(skb)->seq; 207 subflow_req->mp_join = 1; 208 subflow_req->backup = mp_opt.backup; 209 subflow_req->remote_id = mp_opt.join_id; 210 subflow_req->token = mp_opt.token; 211 subflow_req->remote_nonce = mp_opt.nonce; 212 subflow_req->msk = subflow_token_join_request(req); 213 214 /* Can't fall back to TCP in this case. */ 215 if (!subflow_req->msk) { 216 subflow_add_reset_reason(skb, MPTCP_RST_EMPTCP); 217 return -EPERM; 218 } 219 220 if (subflow_use_different_sport(subflow_req->msk, sk_listener)) { 221 pr_debug("syn inet_sport=%d %d", 222 ntohs(inet_sk(sk_listener)->inet_sport), 223 ntohs(inet_sk((struct sock *)subflow_req->msk)->inet_sport)); 224 if (!mptcp_pm_sport_in_anno_list(subflow_req->msk, sk_listener)) { 225 SUBFLOW_REQ_INC_STATS(req, MPTCP_MIB_MISMATCHPORTSYNRX); 226 return -EPERM; 227 } 228 SUBFLOW_REQ_INC_STATS(req, MPTCP_MIB_JOINPORTSYNRX); 229 } 230 231 subflow_req_create_thmac(subflow_req); 232 233 if (unlikely(req->syncookie)) { 234 if (mptcp_can_accept_new_subflow(subflow_req->msk)) 235 subflow_init_req_cookie_join_save(subflow_req, skb); 236 else 237 return -EPERM; 238 } 239 240 pr_debug("token=%u, remote_nonce=%u msk=%p", subflow_req->token, 241 subflow_req->remote_nonce, subflow_req->msk); 242 } 243 244 return 0; 245 } 246 247 int mptcp_subflow_init_cookie_req(struct request_sock *req, 248 const struct sock *sk_listener, 249 struct sk_buff *skb) 250 { 251 struct mptcp_subflow_context *listener = mptcp_subflow_ctx(sk_listener); 252 struct mptcp_subflow_request_sock *subflow_req = mptcp_subflow_rsk(req); 253 struct mptcp_options_received mp_opt; 254 bool opt_mp_capable, opt_mp_join; 255 int err; 256 257 subflow_init_req(req, sk_listener); 258 mptcp_get_options(skb, &mp_opt); 259 260 opt_mp_capable = !!(mp_opt.suboptions & OPTION_MPTCP_MPC_ACK); 261 opt_mp_join = !!(mp_opt.suboptions & OPTION_MPTCP_MPJ_ACK); 262 if (opt_mp_capable && opt_mp_join) 263 return -EINVAL; 264 265 if (opt_mp_capable && listener->request_mptcp) { 266 if (mp_opt.sndr_key == 0) 267 return -EINVAL; 268 269 subflow_req->local_key = mp_opt.rcvr_key; 270 err = mptcp_token_new_request(req); 271 if (err) 272 return err; 273 274 subflow_req->mp_capable = 1; 275 subflow_req->ssn_offset = TCP_SKB_CB(skb)->seq - 1; 276 } else if (opt_mp_join && listener->request_mptcp) { 277 if (!mptcp_token_join_cookie_init_state(subflow_req, skb)) 278 return -EINVAL; 279 280 subflow_req->mp_join = 1; 281 subflow_req->ssn_offset = TCP_SKB_CB(skb)->seq - 1; 282 } 283 284 return 0; 285 } 286 EXPORT_SYMBOL_GPL(mptcp_subflow_init_cookie_req); 287 288 static struct dst_entry *subflow_v4_route_req(const struct sock *sk, 289 struct sk_buff *skb, 290 struct flowi *fl, 291 struct request_sock *req) 292 { 293 struct dst_entry *dst; 294 int err; 295 296 tcp_rsk(req)->is_mptcp = 1; 297 subflow_init_req(req, sk); 298 299 dst = tcp_request_sock_ipv4_ops.route_req(sk, skb, fl, req); 300 if (!dst) 301 return NULL; 302 303 err = subflow_check_req(req, sk, skb); 304 if (err == 0) 305 return dst; 306 307 dst_release(dst); 308 if (!req->syncookie) 309 tcp_request_sock_ops.send_reset(sk, skb); 310 return NULL; 311 } 312 313 static void subflow_prep_synack(const struct sock *sk, struct request_sock *req, 314 struct tcp_fastopen_cookie *foc, 315 enum tcp_synack_type synack_type) 316 { 317 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(sk); 318 struct inet_request_sock *ireq = inet_rsk(req); 319 320 /* clear tstamp_ok, as needed depending on cookie */ 321 if (foc && foc->len > -1) 322 ireq->tstamp_ok = 0; 323 324 if (synack_type == TCP_SYNACK_FASTOPEN) 325 mptcp_fastopen_subflow_synack_set_params(subflow, req); 326 } 327 328 static int subflow_v4_send_synack(const struct sock *sk, struct dst_entry *dst, 329 struct flowi *fl, 330 struct request_sock *req, 331 struct tcp_fastopen_cookie *foc, 332 enum tcp_synack_type synack_type, 333 struct sk_buff *syn_skb) 334 { 335 subflow_prep_synack(sk, req, foc, synack_type); 336 337 return tcp_request_sock_ipv4_ops.send_synack(sk, dst, fl, req, foc, 338 synack_type, syn_skb); 339 } 340 341 #if IS_ENABLED(CONFIG_MPTCP_IPV6) 342 static int subflow_v6_send_synack(const struct sock *sk, struct dst_entry *dst, 343 struct flowi *fl, 344 struct request_sock *req, 345 struct tcp_fastopen_cookie *foc, 346 enum tcp_synack_type synack_type, 347 struct sk_buff *syn_skb) 348 { 349 subflow_prep_synack(sk, req, foc, synack_type); 350 351 return tcp_request_sock_ipv6_ops.send_synack(sk, dst, fl, req, foc, 352 synack_type, syn_skb); 353 } 354 355 static struct dst_entry *subflow_v6_route_req(const struct sock *sk, 356 struct sk_buff *skb, 357 struct flowi *fl, 358 struct request_sock *req) 359 { 360 struct dst_entry *dst; 361 int err; 362 363 tcp_rsk(req)->is_mptcp = 1; 364 subflow_init_req(req, sk); 365 366 dst = tcp_request_sock_ipv6_ops.route_req(sk, skb, fl, req); 367 if (!dst) 368 return NULL; 369 370 err = subflow_check_req(req, sk, skb); 371 if (err == 0) 372 return dst; 373 374 dst_release(dst); 375 if (!req->syncookie) 376 tcp6_request_sock_ops.send_reset(sk, skb); 377 return NULL; 378 } 379 #endif 380 381 /* validate received truncated hmac and create hmac for third ACK */ 382 static bool subflow_thmac_valid(struct mptcp_subflow_context *subflow) 383 { 384 u8 hmac[SHA256_DIGEST_SIZE]; 385 u64 thmac; 386 387 subflow_generate_hmac(subflow->remote_key, subflow->local_key, 388 subflow->remote_nonce, subflow->local_nonce, 389 hmac); 390 391 thmac = get_unaligned_be64(hmac); 392 pr_debug("subflow=%p, token=%u, thmac=%llu, subflow->thmac=%llu\n", 393 subflow, subflow->token, thmac, subflow->thmac); 394 395 return thmac == subflow->thmac; 396 } 397 398 void mptcp_subflow_reset(struct sock *ssk) 399 { 400 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk); 401 struct sock *sk = subflow->conn; 402 403 /* mptcp_mp_fail_no_response() can reach here on an already closed 404 * socket 405 */ 406 if (ssk->sk_state == TCP_CLOSE) 407 return; 408 409 /* must hold: tcp_done() could drop last reference on parent */ 410 sock_hold(sk); 411 412 tcp_send_active_reset(ssk, GFP_ATOMIC); 413 tcp_done(ssk); 414 if (!test_and_set_bit(MPTCP_WORK_CLOSE_SUBFLOW, &mptcp_sk(sk)->flags)) 415 mptcp_schedule_work(sk); 416 417 sock_put(sk); 418 } 419 420 static bool subflow_use_different_dport(struct mptcp_sock *msk, const struct sock *sk) 421 { 422 return inet_sk(sk)->inet_dport != inet_sk((struct sock *)msk)->inet_dport; 423 } 424 425 void __mptcp_sync_state(struct sock *sk, int state) 426 { 427 struct mptcp_subflow_context *subflow; 428 struct mptcp_sock *msk = mptcp_sk(sk); 429 struct sock *ssk = msk->first; 430 431 subflow = mptcp_subflow_ctx(ssk); 432 __mptcp_propagate_sndbuf(sk, ssk); 433 if (!msk->rcvspace_init) 434 mptcp_rcv_space_init(msk, ssk); 435 436 if (sk->sk_state == TCP_SYN_SENT) { 437 /* subflow->idsn is always available is TCP_SYN_SENT state, 438 * even for the FASTOPEN scenarios 439 */ 440 WRITE_ONCE(msk->write_seq, subflow->idsn + 1); 441 WRITE_ONCE(msk->snd_nxt, msk->write_seq); 442 mptcp_set_state(sk, state); 443 sk->sk_state_change(sk); 444 } 445 } 446 447 static void subflow_set_remote_key(struct mptcp_sock *msk, 448 struct mptcp_subflow_context *subflow, 449 const struct mptcp_options_received *mp_opt) 450 { 451 /* active MPC subflow will reach here multiple times: 452 * at subflow_finish_connect() time and at 4th ack time 453 */ 454 if (subflow->remote_key_valid) 455 return; 456 457 subflow->remote_key_valid = 1; 458 subflow->remote_key = mp_opt->sndr_key; 459 mptcp_crypto_key_sha(subflow->remote_key, NULL, &subflow->iasn); 460 subflow->iasn++; 461 462 WRITE_ONCE(msk->remote_key, subflow->remote_key); 463 WRITE_ONCE(msk->ack_seq, subflow->iasn); 464 WRITE_ONCE(msk->can_ack, true); 465 atomic64_set(&msk->rcv_wnd_sent, subflow->iasn); 466 } 467 468 static void mptcp_propagate_state(struct sock *sk, struct sock *ssk, 469 struct mptcp_subflow_context *subflow, 470 const struct mptcp_options_received *mp_opt) 471 { 472 struct mptcp_sock *msk = mptcp_sk(sk); 473 474 mptcp_data_lock(sk); 475 if (mp_opt) { 476 /* Options are available only in the non fallback cases 477 * avoid updating rx path fields otherwise 478 */ 479 WRITE_ONCE(msk->snd_una, subflow->idsn + 1); 480 WRITE_ONCE(msk->wnd_end, subflow->idsn + 1 + tcp_sk(ssk)->snd_wnd); 481 subflow_set_remote_key(msk, subflow, mp_opt); 482 } 483 484 if (!sock_owned_by_user(sk)) { 485 __mptcp_sync_state(sk, ssk->sk_state); 486 } else { 487 msk->pending_state = ssk->sk_state; 488 __set_bit(MPTCP_SYNC_STATE, &msk->cb_flags); 489 } 490 mptcp_data_unlock(sk); 491 } 492 493 static void subflow_finish_connect(struct sock *sk, const struct sk_buff *skb) 494 { 495 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(sk); 496 struct mptcp_options_received mp_opt; 497 struct sock *parent = subflow->conn; 498 struct mptcp_sock *msk; 499 500 subflow->icsk_af_ops->sk_rx_dst_set(sk, skb); 501 502 /* be sure no special action on any packet other than syn-ack */ 503 if (subflow->conn_finished) 504 return; 505 506 msk = mptcp_sk(parent); 507 subflow->rel_write_seq = 1; 508 subflow->conn_finished = 1; 509 subflow->ssn_offset = TCP_SKB_CB(skb)->seq; 510 pr_debug("subflow=%p synack seq=%x", subflow, subflow->ssn_offset); 511 512 mptcp_get_options(skb, &mp_opt); 513 if (subflow->request_mptcp) { 514 if (!(mp_opt.suboptions & OPTION_MPTCP_MPC_SYNACK)) { 515 MPTCP_INC_STATS(sock_net(sk), 516 MPTCP_MIB_MPCAPABLEACTIVEFALLBACK); 517 mptcp_do_fallback(sk); 518 pr_fallback(msk); 519 goto fallback; 520 } 521 522 if (mp_opt.suboptions & OPTION_MPTCP_CSUMREQD) 523 WRITE_ONCE(msk->csum_enabled, true); 524 if (mp_opt.deny_join_id0) 525 WRITE_ONCE(msk->pm.remote_deny_join_id0, true); 526 subflow->mp_capable = 1; 527 MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_MPCAPABLEACTIVEACK); 528 mptcp_finish_connect(sk); 529 mptcp_propagate_state(parent, sk, subflow, &mp_opt); 530 } else if (subflow->request_join) { 531 u8 hmac[SHA256_DIGEST_SIZE]; 532 533 if (!(mp_opt.suboptions & OPTION_MPTCP_MPJ_SYNACK)) { 534 subflow->reset_reason = MPTCP_RST_EMPTCP; 535 goto do_reset; 536 } 537 538 subflow->backup = mp_opt.backup; 539 subflow->thmac = mp_opt.thmac; 540 subflow->remote_nonce = mp_opt.nonce; 541 WRITE_ONCE(subflow->remote_id, mp_opt.join_id); 542 pr_debug("subflow=%p, thmac=%llu, remote_nonce=%u backup=%d", 543 subflow, subflow->thmac, subflow->remote_nonce, 544 subflow->backup); 545 546 if (!subflow_thmac_valid(subflow)) { 547 MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_JOINACKMAC); 548 subflow->reset_reason = MPTCP_RST_EMPTCP; 549 goto do_reset; 550 } 551 552 if (!mptcp_finish_join(sk)) 553 goto do_reset; 554 555 subflow_generate_hmac(subflow->local_key, subflow->remote_key, 556 subflow->local_nonce, 557 subflow->remote_nonce, 558 hmac); 559 memcpy(subflow->hmac, hmac, MPTCPOPT_HMAC_LEN); 560 561 subflow->mp_join = 1; 562 MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_JOINSYNACKRX); 563 564 if (subflow->backup) 565 MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_JOINSYNACKBACKUPRX); 566 567 if (subflow_use_different_dport(msk, sk)) { 568 pr_debug("synack inet_dport=%d %d", 569 ntohs(inet_sk(sk)->inet_dport), 570 ntohs(inet_sk(parent)->inet_dport)); 571 MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_JOINPORTSYNACKRX); 572 } 573 } else if (mptcp_check_fallback(sk)) { 574 fallback: 575 mptcp_propagate_state(parent, sk, subflow, NULL); 576 } 577 return; 578 579 do_reset: 580 subflow->reset_transient = 0; 581 mptcp_subflow_reset(sk); 582 } 583 584 static void subflow_set_local_id(struct mptcp_subflow_context *subflow, int local_id) 585 { 586 WARN_ON_ONCE(local_id < 0 || local_id > 255); 587 WRITE_ONCE(subflow->local_id, local_id); 588 } 589 590 static int subflow_chk_local_id(struct sock *sk) 591 { 592 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(sk); 593 struct mptcp_sock *msk = mptcp_sk(subflow->conn); 594 int err; 595 596 if (likely(subflow->local_id >= 0)) 597 return 0; 598 599 err = mptcp_pm_get_local_id(msk, (struct sock_common *)sk); 600 if (err < 0) 601 return err; 602 603 subflow_set_local_id(subflow, err); 604 return 0; 605 } 606 607 static int subflow_rebuild_header(struct sock *sk) 608 { 609 int err = subflow_chk_local_id(sk); 610 611 if (unlikely(err < 0)) 612 return err; 613 614 return inet_sk_rebuild_header(sk); 615 } 616 617 #if IS_ENABLED(CONFIG_MPTCP_IPV6) 618 static int subflow_v6_rebuild_header(struct sock *sk) 619 { 620 int err = subflow_chk_local_id(sk); 621 622 if (unlikely(err < 0)) 623 return err; 624 625 return inet6_sk_rebuild_header(sk); 626 } 627 #endif 628 629 static struct request_sock_ops mptcp_subflow_v4_request_sock_ops __ro_after_init; 630 static struct tcp_request_sock_ops subflow_request_sock_ipv4_ops __ro_after_init; 631 632 static int subflow_v4_conn_request(struct sock *sk, struct sk_buff *skb) 633 { 634 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(sk); 635 636 pr_debug("subflow=%p", subflow); 637 638 /* Never answer to SYNs sent to broadcast or multicast */ 639 if (skb_rtable(skb)->rt_flags & (RTCF_BROADCAST | RTCF_MULTICAST)) 640 goto drop; 641 642 return tcp_conn_request(&mptcp_subflow_v4_request_sock_ops, 643 &subflow_request_sock_ipv4_ops, 644 sk, skb); 645 drop: 646 tcp_listendrop(sk); 647 return 0; 648 } 649 650 static void subflow_v4_req_destructor(struct request_sock *req) 651 { 652 subflow_req_destructor(req); 653 tcp_request_sock_ops.destructor(req); 654 } 655 656 #if IS_ENABLED(CONFIG_MPTCP_IPV6) 657 static struct request_sock_ops mptcp_subflow_v6_request_sock_ops __ro_after_init; 658 static struct tcp_request_sock_ops subflow_request_sock_ipv6_ops __ro_after_init; 659 static struct inet_connection_sock_af_ops subflow_v6_specific __ro_after_init; 660 static struct inet_connection_sock_af_ops subflow_v6m_specific __ro_after_init; 661 static struct proto tcpv6_prot_override __ro_after_init; 662 663 static int subflow_v6_conn_request(struct sock *sk, struct sk_buff *skb) 664 { 665 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(sk); 666 667 pr_debug("subflow=%p", subflow); 668 669 if (skb->protocol == htons(ETH_P_IP)) 670 return subflow_v4_conn_request(sk, skb); 671 672 if (!ipv6_unicast_destination(skb)) 673 goto drop; 674 675 if (ipv6_addr_v4mapped(&ipv6_hdr(skb)->saddr)) { 676 __IP6_INC_STATS(sock_net(sk), NULL, IPSTATS_MIB_INHDRERRORS); 677 return 0; 678 } 679 680 return tcp_conn_request(&mptcp_subflow_v6_request_sock_ops, 681 &subflow_request_sock_ipv6_ops, sk, skb); 682 683 drop: 684 tcp_listendrop(sk); 685 return 0; /* don't send reset */ 686 } 687 688 static void subflow_v6_req_destructor(struct request_sock *req) 689 { 690 subflow_req_destructor(req); 691 tcp6_request_sock_ops.destructor(req); 692 } 693 #endif 694 695 struct request_sock *mptcp_subflow_reqsk_alloc(const struct request_sock_ops *ops, 696 struct sock *sk_listener, 697 bool attach_listener) 698 { 699 if (ops->family == AF_INET) 700 ops = &mptcp_subflow_v4_request_sock_ops; 701 #if IS_ENABLED(CONFIG_MPTCP_IPV6) 702 else if (ops->family == AF_INET6) 703 ops = &mptcp_subflow_v6_request_sock_ops; 704 #endif 705 706 return inet_reqsk_alloc(ops, sk_listener, attach_listener); 707 } 708 EXPORT_SYMBOL(mptcp_subflow_reqsk_alloc); 709 710 /* validate hmac received in third ACK */ 711 static bool subflow_hmac_valid(const struct request_sock *req, 712 const struct mptcp_options_received *mp_opt) 713 { 714 const struct mptcp_subflow_request_sock *subflow_req; 715 u8 hmac[SHA256_DIGEST_SIZE]; 716 struct mptcp_sock *msk; 717 718 subflow_req = mptcp_subflow_rsk(req); 719 msk = subflow_req->msk; 720 if (!msk) 721 return false; 722 723 subflow_generate_hmac(msk->remote_key, msk->local_key, 724 subflow_req->remote_nonce, 725 subflow_req->local_nonce, hmac); 726 727 return !crypto_memneq(hmac, mp_opt->hmac, MPTCPOPT_HMAC_LEN); 728 } 729 730 static void subflow_ulp_fallback(struct sock *sk, 731 struct mptcp_subflow_context *old_ctx) 732 { 733 struct inet_connection_sock *icsk = inet_csk(sk); 734 735 mptcp_subflow_tcp_fallback(sk, old_ctx); 736 icsk->icsk_ulp_ops = NULL; 737 rcu_assign_pointer(icsk->icsk_ulp_data, NULL); 738 tcp_sk(sk)->is_mptcp = 0; 739 740 mptcp_subflow_ops_undo_override(sk); 741 } 742 743 void mptcp_subflow_drop_ctx(struct sock *ssk) 744 { 745 struct mptcp_subflow_context *ctx = mptcp_subflow_ctx(ssk); 746 747 if (!ctx) 748 return; 749 750 list_del(&mptcp_subflow_ctx(ssk)->node); 751 if (inet_csk(ssk)->icsk_ulp_ops) { 752 subflow_ulp_fallback(ssk, ctx); 753 if (ctx->conn) 754 sock_put(ctx->conn); 755 } 756 757 kfree_rcu(ctx, rcu); 758 } 759 760 void __mptcp_subflow_fully_established(struct mptcp_sock *msk, 761 struct mptcp_subflow_context *subflow, 762 const struct mptcp_options_received *mp_opt) 763 { 764 subflow_set_remote_key(msk, subflow, mp_opt); 765 subflow->fully_established = 1; 766 WRITE_ONCE(msk->fully_established, true); 767 768 if (subflow->is_mptfo) 769 __mptcp_fastopen_gen_msk_ackseq(msk, subflow, mp_opt); 770 } 771 772 static struct sock *subflow_syn_recv_sock(const struct sock *sk, 773 struct sk_buff *skb, 774 struct request_sock *req, 775 struct dst_entry *dst, 776 struct request_sock *req_unhash, 777 bool *own_req) 778 { 779 struct mptcp_subflow_context *listener = mptcp_subflow_ctx(sk); 780 struct mptcp_subflow_request_sock *subflow_req; 781 struct mptcp_options_received mp_opt; 782 bool fallback, fallback_is_fatal; 783 struct mptcp_sock *owner; 784 struct sock *child; 785 786 pr_debug("listener=%p, req=%p, conn=%p", listener, req, listener->conn); 787 788 /* After child creation we must look for MPC even when options 789 * are not parsed 790 */ 791 mp_opt.suboptions = 0; 792 793 /* hopefully temporary handling for MP_JOIN+syncookie */ 794 subflow_req = mptcp_subflow_rsk(req); 795 fallback_is_fatal = tcp_rsk(req)->is_mptcp && subflow_req->mp_join; 796 fallback = !tcp_rsk(req)->is_mptcp; 797 if (fallback) 798 goto create_child; 799 800 /* if the sk is MP_CAPABLE, we try to fetch the client key */ 801 if (subflow_req->mp_capable) { 802 /* we can receive and accept an in-window, out-of-order pkt, 803 * which may not carry the MP_CAPABLE opt even on mptcp enabled 804 * paths: always try to extract the peer key, and fallback 805 * for packets missing it. 806 * Even OoO DSS packets coming legitly after dropped or 807 * reordered MPC will cause fallback, but we don't have other 808 * options. 809 */ 810 mptcp_get_options(skb, &mp_opt); 811 if (!(mp_opt.suboptions & 812 (OPTION_MPTCP_MPC_SYN | OPTION_MPTCP_MPC_ACK))) 813 fallback = true; 814 815 } else if (subflow_req->mp_join) { 816 mptcp_get_options(skb, &mp_opt); 817 if (!(mp_opt.suboptions & OPTION_MPTCP_MPJ_ACK) || 818 !subflow_hmac_valid(req, &mp_opt) || 819 !mptcp_can_accept_new_subflow(subflow_req->msk)) { 820 SUBFLOW_REQ_INC_STATS(req, MPTCP_MIB_JOINACKMAC); 821 fallback = true; 822 } 823 } 824 825 create_child: 826 child = listener->icsk_af_ops->syn_recv_sock(sk, skb, req, dst, 827 req_unhash, own_req); 828 829 if (child && *own_req) { 830 struct mptcp_subflow_context *ctx = mptcp_subflow_ctx(child); 831 832 tcp_rsk(req)->drop_req = false; 833 834 /* we need to fallback on ctx allocation failure and on pre-reqs 835 * checking above. In the latter scenario we additionally need 836 * to reset the context to non MPTCP status. 837 */ 838 if (!ctx || fallback) { 839 if (fallback_is_fatal) { 840 subflow_add_reset_reason(skb, MPTCP_RST_EMPTCP); 841 goto dispose_child; 842 } 843 goto fallback; 844 } 845 846 /* ssk inherits options of listener sk */ 847 ctx->setsockopt_seq = listener->setsockopt_seq; 848 849 if (ctx->mp_capable) { 850 ctx->conn = mptcp_sk_clone_init(listener->conn, &mp_opt, child, req); 851 if (!ctx->conn) 852 goto fallback; 853 854 ctx->subflow_id = 1; 855 owner = mptcp_sk(ctx->conn); 856 mptcp_pm_new_connection(owner, child, 1); 857 858 /* with OoO packets we can reach here without ingress 859 * mpc option 860 */ 861 if (mp_opt.suboptions & OPTION_MPTCP_MPC_ACK) { 862 mptcp_pm_fully_established(owner, child); 863 ctx->pm_notified = 1; 864 } 865 } else if (ctx->mp_join) { 866 owner = subflow_req->msk; 867 if (!owner) { 868 subflow_add_reset_reason(skb, MPTCP_RST_EPROHIBIT); 869 goto dispose_child; 870 } 871 872 /* move the msk reference ownership to the subflow */ 873 subflow_req->msk = NULL; 874 ctx->conn = (struct sock *)owner; 875 876 if (subflow_use_different_sport(owner, sk)) { 877 pr_debug("ack inet_sport=%d %d", 878 ntohs(inet_sk(sk)->inet_sport), 879 ntohs(inet_sk((struct sock *)owner)->inet_sport)); 880 if (!mptcp_pm_sport_in_anno_list(owner, sk)) { 881 SUBFLOW_REQ_INC_STATS(req, MPTCP_MIB_MISMATCHPORTACKRX); 882 goto dispose_child; 883 } 884 SUBFLOW_REQ_INC_STATS(req, MPTCP_MIB_JOINPORTACKRX); 885 } 886 887 if (!mptcp_finish_join(child)) 888 goto dispose_child; 889 890 SUBFLOW_REQ_INC_STATS(req, MPTCP_MIB_JOINACKRX); 891 tcp_rsk(req)->drop_req = true; 892 } 893 } 894 895 /* check for expected invariant - should never trigger, just help 896 * catching eariler subtle bugs 897 */ 898 WARN_ON_ONCE(child && *own_req && tcp_sk(child)->is_mptcp && 899 (!mptcp_subflow_ctx(child) || 900 !mptcp_subflow_ctx(child)->conn)); 901 return child; 902 903 dispose_child: 904 mptcp_subflow_drop_ctx(child); 905 tcp_rsk(req)->drop_req = true; 906 inet_csk_prepare_for_destroy_sock(child); 907 tcp_done(child); 908 req->rsk_ops->send_reset(sk, skb); 909 910 /* The last child reference will be released by the caller */ 911 return child; 912 913 fallback: 914 if (fallback) 915 SUBFLOW_REQ_INC_STATS(req, MPTCP_MIB_MPCAPABLEPASSIVEFALLBACK); 916 mptcp_subflow_drop_ctx(child); 917 return child; 918 } 919 920 static struct inet_connection_sock_af_ops subflow_specific __ro_after_init; 921 static struct proto tcp_prot_override __ro_after_init; 922 923 enum mapping_status { 924 MAPPING_OK, 925 MAPPING_INVALID, 926 MAPPING_EMPTY, 927 MAPPING_DATA_FIN, 928 MAPPING_DUMMY, 929 MAPPING_BAD_CSUM 930 }; 931 932 static void dbg_bad_map(struct mptcp_subflow_context *subflow, u32 ssn) 933 { 934 pr_debug("Bad mapping: ssn=%d map_seq=%d map_data_len=%d", 935 ssn, subflow->map_subflow_seq, subflow->map_data_len); 936 } 937 938 static bool skb_is_fully_mapped(struct sock *ssk, struct sk_buff *skb) 939 { 940 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk); 941 unsigned int skb_consumed; 942 943 skb_consumed = tcp_sk(ssk)->copied_seq - TCP_SKB_CB(skb)->seq; 944 if (WARN_ON_ONCE(skb_consumed >= skb->len)) 945 return true; 946 947 return skb->len - skb_consumed <= subflow->map_data_len - 948 mptcp_subflow_get_map_offset(subflow); 949 } 950 951 static bool validate_mapping(struct sock *ssk, struct sk_buff *skb) 952 { 953 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk); 954 u32 ssn = tcp_sk(ssk)->copied_seq - subflow->ssn_offset; 955 956 if (unlikely(before(ssn, subflow->map_subflow_seq))) { 957 /* Mapping covers data later in the subflow stream, 958 * currently unsupported. 959 */ 960 dbg_bad_map(subflow, ssn); 961 return false; 962 } 963 if (unlikely(!before(ssn, subflow->map_subflow_seq + 964 subflow->map_data_len))) { 965 /* Mapping does covers past subflow data, invalid */ 966 dbg_bad_map(subflow, ssn); 967 return false; 968 } 969 return true; 970 } 971 972 static enum mapping_status validate_data_csum(struct sock *ssk, struct sk_buff *skb, 973 bool csum_reqd) 974 { 975 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk); 976 u32 offset, seq, delta; 977 __sum16 csum; 978 int len; 979 980 if (!csum_reqd) 981 return MAPPING_OK; 982 983 /* mapping already validated on previous traversal */ 984 if (subflow->map_csum_len == subflow->map_data_len) 985 return MAPPING_OK; 986 987 /* traverse the receive queue, ensuring it contains a full 988 * DSS mapping and accumulating the related csum. 989 * Preserve the accoumlate csum across multiple calls, to compute 990 * the csum only once 991 */ 992 delta = subflow->map_data_len - subflow->map_csum_len; 993 for (;;) { 994 seq = tcp_sk(ssk)->copied_seq + subflow->map_csum_len; 995 offset = seq - TCP_SKB_CB(skb)->seq; 996 997 /* if the current skb has not been accounted yet, csum its contents 998 * up to the amount covered by the current DSS 999 */ 1000 if (offset < skb->len) { 1001 __wsum csum; 1002 1003 len = min(skb->len - offset, delta); 1004 csum = skb_checksum(skb, offset, len, 0); 1005 subflow->map_data_csum = csum_block_add(subflow->map_data_csum, csum, 1006 subflow->map_csum_len); 1007 1008 delta -= len; 1009 subflow->map_csum_len += len; 1010 } 1011 if (delta == 0) 1012 break; 1013 1014 if (skb_queue_is_last(&ssk->sk_receive_queue, skb)) { 1015 /* if this subflow is closed, the partial mapping 1016 * will be never completed; flush the pending skbs, so 1017 * that subflow_sched_work_if_closed() can kick in 1018 */ 1019 if (unlikely(ssk->sk_state == TCP_CLOSE)) 1020 while ((skb = skb_peek(&ssk->sk_receive_queue))) 1021 sk_eat_skb(ssk, skb); 1022 1023 /* not enough data to validate the csum */ 1024 return MAPPING_EMPTY; 1025 } 1026 1027 /* the DSS mapping for next skbs will be validated later, 1028 * when a get_mapping_status call will process such skb 1029 */ 1030 skb = skb->next; 1031 } 1032 1033 /* note that 'map_data_len' accounts only for the carried data, does 1034 * not include the eventual seq increment due to the data fin, 1035 * while the pseudo header requires the original DSS data len, 1036 * including that 1037 */ 1038 csum = __mptcp_make_csum(subflow->map_seq, 1039 subflow->map_subflow_seq, 1040 subflow->map_data_len + subflow->map_data_fin, 1041 subflow->map_data_csum); 1042 if (unlikely(csum)) { 1043 MPTCP_INC_STATS(sock_net(ssk), MPTCP_MIB_DATACSUMERR); 1044 return MAPPING_BAD_CSUM; 1045 } 1046 1047 subflow->valid_csum_seen = 1; 1048 return MAPPING_OK; 1049 } 1050 1051 static enum mapping_status get_mapping_status(struct sock *ssk, 1052 struct mptcp_sock *msk) 1053 { 1054 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk); 1055 bool csum_reqd = READ_ONCE(msk->csum_enabled); 1056 struct mptcp_ext *mpext; 1057 struct sk_buff *skb; 1058 u16 data_len; 1059 u64 map_seq; 1060 1061 skb = skb_peek(&ssk->sk_receive_queue); 1062 if (!skb) 1063 return MAPPING_EMPTY; 1064 1065 if (mptcp_check_fallback(ssk)) 1066 return MAPPING_DUMMY; 1067 1068 mpext = mptcp_get_ext(skb); 1069 if (!mpext || !mpext->use_map) { 1070 if (!subflow->map_valid && !skb->len) { 1071 /* the TCP stack deliver 0 len FIN pkt to the receive 1072 * queue, that is the only 0len pkts ever expected here, 1073 * and we can admit no mapping only for 0 len pkts 1074 */ 1075 if (!(TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN)) 1076 WARN_ONCE(1, "0len seq %d:%d flags %x", 1077 TCP_SKB_CB(skb)->seq, 1078 TCP_SKB_CB(skb)->end_seq, 1079 TCP_SKB_CB(skb)->tcp_flags); 1080 sk_eat_skb(ssk, skb); 1081 return MAPPING_EMPTY; 1082 } 1083 1084 if (!subflow->map_valid) 1085 return MAPPING_INVALID; 1086 1087 goto validate_seq; 1088 } 1089 1090 trace_get_mapping_status(mpext); 1091 1092 data_len = mpext->data_len; 1093 if (data_len == 0) { 1094 pr_debug("infinite mapping received"); 1095 MPTCP_INC_STATS(sock_net(ssk), MPTCP_MIB_INFINITEMAPRX); 1096 subflow->map_data_len = 0; 1097 return MAPPING_INVALID; 1098 } 1099 1100 if (mpext->data_fin == 1) { 1101 if (data_len == 1) { 1102 bool updated = mptcp_update_rcv_data_fin(msk, mpext->data_seq, 1103 mpext->dsn64); 1104 pr_debug("DATA_FIN with no payload seq=%llu", mpext->data_seq); 1105 if (subflow->map_valid) { 1106 /* A DATA_FIN might arrive in a DSS 1107 * option before the previous mapping 1108 * has been fully consumed. Continue 1109 * handling the existing mapping. 1110 */ 1111 skb_ext_del(skb, SKB_EXT_MPTCP); 1112 return MAPPING_OK; 1113 } else { 1114 if (updated) 1115 mptcp_schedule_work((struct sock *)msk); 1116 1117 return MAPPING_DATA_FIN; 1118 } 1119 } else { 1120 u64 data_fin_seq = mpext->data_seq + data_len - 1; 1121 1122 /* If mpext->data_seq is a 32-bit value, data_fin_seq 1123 * must also be limited to 32 bits. 1124 */ 1125 if (!mpext->dsn64) 1126 data_fin_seq &= GENMASK_ULL(31, 0); 1127 1128 mptcp_update_rcv_data_fin(msk, data_fin_seq, mpext->dsn64); 1129 pr_debug("DATA_FIN with mapping seq=%llu dsn64=%d", 1130 data_fin_seq, mpext->dsn64); 1131 } 1132 1133 /* Adjust for DATA_FIN using 1 byte of sequence space */ 1134 data_len--; 1135 } 1136 1137 map_seq = mptcp_expand_seq(READ_ONCE(msk->ack_seq), mpext->data_seq, mpext->dsn64); 1138 WRITE_ONCE(mptcp_sk(subflow->conn)->use_64bit_ack, !!mpext->dsn64); 1139 1140 if (subflow->map_valid) { 1141 /* Allow replacing only with an identical map */ 1142 if (subflow->map_seq == map_seq && 1143 subflow->map_subflow_seq == mpext->subflow_seq && 1144 subflow->map_data_len == data_len && 1145 subflow->map_csum_reqd == mpext->csum_reqd) { 1146 skb_ext_del(skb, SKB_EXT_MPTCP); 1147 goto validate_csum; 1148 } 1149 1150 /* If this skb data are fully covered by the current mapping, 1151 * the new map would need caching, which is not supported 1152 */ 1153 if (skb_is_fully_mapped(ssk, skb)) { 1154 MPTCP_INC_STATS(sock_net(ssk), MPTCP_MIB_DSSNOMATCH); 1155 return MAPPING_INVALID; 1156 } 1157 1158 /* will validate the next map after consuming the current one */ 1159 goto validate_csum; 1160 } 1161 1162 subflow->map_seq = map_seq; 1163 subflow->map_subflow_seq = mpext->subflow_seq; 1164 subflow->map_data_len = data_len; 1165 subflow->map_valid = 1; 1166 subflow->map_data_fin = mpext->data_fin; 1167 subflow->mpc_map = mpext->mpc_map; 1168 subflow->map_csum_reqd = mpext->csum_reqd; 1169 subflow->map_csum_len = 0; 1170 subflow->map_data_csum = csum_unfold(mpext->csum); 1171 1172 /* Cfr RFC 8684 Section 3.3.0 */ 1173 if (unlikely(subflow->map_csum_reqd != csum_reqd)) 1174 return MAPPING_INVALID; 1175 1176 pr_debug("new map seq=%llu subflow_seq=%u data_len=%u csum=%d:%u", 1177 subflow->map_seq, subflow->map_subflow_seq, 1178 subflow->map_data_len, subflow->map_csum_reqd, 1179 subflow->map_data_csum); 1180 1181 validate_seq: 1182 /* we revalidate valid mapping on new skb, because we must ensure 1183 * the current skb is completely covered by the available mapping 1184 */ 1185 if (!validate_mapping(ssk, skb)) { 1186 MPTCP_INC_STATS(sock_net(ssk), MPTCP_MIB_DSSTCPMISMATCH); 1187 return MAPPING_INVALID; 1188 } 1189 1190 skb_ext_del(skb, SKB_EXT_MPTCP); 1191 1192 validate_csum: 1193 return validate_data_csum(ssk, skb, csum_reqd); 1194 } 1195 1196 static void mptcp_subflow_discard_data(struct sock *ssk, struct sk_buff *skb, 1197 u64 limit) 1198 { 1199 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk); 1200 bool fin = TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN; 1201 struct tcp_sock *tp = tcp_sk(ssk); 1202 u32 offset, incr, avail_len; 1203 1204 offset = tp->copied_seq - TCP_SKB_CB(skb)->seq; 1205 if (WARN_ON_ONCE(offset > skb->len)) 1206 goto out; 1207 1208 avail_len = skb->len - offset; 1209 incr = limit >= avail_len ? avail_len + fin : limit; 1210 1211 pr_debug("discarding=%d len=%d offset=%d seq=%d", incr, skb->len, 1212 offset, subflow->map_subflow_seq); 1213 MPTCP_INC_STATS(sock_net(ssk), MPTCP_MIB_DUPDATA); 1214 tcp_sk(ssk)->copied_seq += incr; 1215 1216 out: 1217 if (!before(tcp_sk(ssk)->copied_seq, TCP_SKB_CB(skb)->end_seq)) 1218 sk_eat_skb(ssk, skb); 1219 if (mptcp_subflow_get_map_offset(subflow) >= subflow->map_data_len) 1220 subflow->map_valid = 0; 1221 } 1222 1223 /* sched mptcp worker to remove the subflow if no more data is pending */ 1224 static void subflow_sched_work_if_closed(struct mptcp_sock *msk, struct sock *ssk) 1225 { 1226 if (likely(ssk->sk_state != TCP_CLOSE)) 1227 return; 1228 1229 if (skb_queue_empty(&ssk->sk_receive_queue) && 1230 !test_and_set_bit(MPTCP_WORK_CLOSE_SUBFLOW, &msk->flags)) 1231 mptcp_schedule_work((struct sock *)msk); 1232 } 1233 1234 static bool subflow_can_fallback(struct mptcp_subflow_context *subflow) 1235 { 1236 struct mptcp_sock *msk = mptcp_sk(subflow->conn); 1237 1238 if (subflow->mp_join) 1239 return false; 1240 else if (READ_ONCE(msk->csum_enabled)) 1241 return !subflow->valid_csum_seen; 1242 else 1243 return !subflow->fully_established; 1244 } 1245 1246 static void mptcp_subflow_fail(struct mptcp_sock *msk, struct sock *ssk) 1247 { 1248 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk); 1249 unsigned long fail_tout; 1250 1251 /* greceful failure can happen only on the MPC subflow */ 1252 if (WARN_ON_ONCE(ssk != READ_ONCE(msk->first))) 1253 return; 1254 1255 /* since the close timeout take precedence on the fail one, 1256 * no need to start the latter when the first is already set 1257 */ 1258 if (sock_flag((struct sock *)msk, SOCK_DEAD)) 1259 return; 1260 1261 /* we don't need extreme accuracy here, use a zero fail_tout as special 1262 * value meaning no fail timeout at all; 1263 */ 1264 fail_tout = jiffies + TCP_RTO_MAX; 1265 if (!fail_tout) 1266 fail_tout = 1; 1267 WRITE_ONCE(subflow->fail_tout, fail_tout); 1268 tcp_send_ack(ssk); 1269 1270 mptcp_reset_tout_timer(msk, subflow->fail_tout); 1271 } 1272 1273 static bool subflow_check_data_avail(struct sock *ssk) 1274 { 1275 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk); 1276 enum mapping_status status; 1277 struct mptcp_sock *msk; 1278 struct sk_buff *skb; 1279 1280 if (!skb_peek(&ssk->sk_receive_queue)) 1281 WRITE_ONCE(subflow->data_avail, MPTCP_SUBFLOW_NODATA); 1282 if (subflow->data_avail) 1283 return true; 1284 1285 msk = mptcp_sk(subflow->conn); 1286 for (;;) { 1287 u64 ack_seq; 1288 u64 old_ack; 1289 1290 status = get_mapping_status(ssk, msk); 1291 trace_subflow_check_data_avail(status, skb_peek(&ssk->sk_receive_queue)); 1292 if (unlikely(status == MAPPING_INVALID || status == MAPPING_DUMMY || 1293 status == MAPPING_BAD_CSUM)) 1294 goto fallback; 1295 1296 if (status != MAPPING_OK) 1297 goto no_data; 1298 1299 skb = skb_peek(&ssk->sk_receive_queue); 1300 if (WARN_ON_ONCE(!skb)) 1301 goto no_data; 1302 1303 if (unlikely(!READ_ONCE(msk->can_ack))) 1304 goto fallback; 1305 1306 old_ack = READ_ONCE(msk->ack_seq); 1307 ack_seq = mptcp_subflow_get_mapped_dsn(subflow); 1308 pr_debug("msk ack_seq=%llx subflow ack_seq=%llx", old_ack, 1309 ack_seq); 1310 if (unlikely(before64(ack_seq, old_ack))) { 1311 mptcp_subflow_discard_data(ssk, skb, old_ack - ack_seq); 1312 continue; 1313 } 1314 1315 WRITE_ONCE(subflow->data_avail, MPTCP_SUBFLOW_DATA_AVAIL); 1316 break; 1317 } 1318 return true; 1319 1320 no_data: 1321 subflow_sched_work_if_closed(msk, ssk); 1322 return false; 1323 1324 fallback: 1325 if (!__mptcp_check_fallback(msk)) { 1326 /* RFC 8684 section 3.7. */ 1327 if (status == MAPPING_BAD_CSUM && 1328 (subflow->mp_join || subflow->valid_csum_seen)) { 1329 subflow->send_mp_fail = 1; 1330 1331 if (!READ_ONCE(msk->allow_infinite_fallback)) { 1332 subflow->reset_transient = 0; 1333 subflow->reset_reason = MPTCP_RST_EMIDDLEBOX; 1334 goto reset; 1335 } 1336 mptcp_subflow_fail(msk, ssk); 1337 WRITE_ONCE(subflow->data_avail, MPTCP_SUBFLOW_DATA_AVAIL); 1338 return true; 1339 } 1340 1341 if (!subflow_can_fallback(subflow) && subflow->map_data_len) { 1342 /* fatal protocol error, close the socket. 1343 * subflow_error_report() will introduce the appropriate barriers 1344 */ 1345 subflow->reset_transient = 0; 1346 subflow->reset_reason = MPTCP_RST_EMPTCP; 1347 1348 reset: 1349 WRITE_ONCE(ssk->sk_err, EBADMSG); 1350 tcp_set_state(ssk, TCP_CLOSE); 1351 while ((skb = skb_peek(&ssk->sk_receive_queue))) 1352 sk_eat_skb(ssk, skb); 1353 tcp_send_active_reset(ssk, GFP_ATOMIC); 1354 WRITE_ONCE(subflow->data_avail, MPTCP_SUBFLOW_NODATA); 1355 return false; 1356 } 1357 1358 mptcp_do_fallback(ssk); 1359 } 1360 1361 skb = skb_peek(&ssk->sk_receive_queue); 1362 subflow->map_valid = 1; 1363 subflow->map_seq = READ_ONCE(msk->ack_seq); 1364 subflow->map_data_len = skb->len; 1365 subflow->map_subflow_seq = tcp_sk(ssk)->copied_seq - subflow->ssn_offset; 1366 WRITE_ONCE(subflow->data_avail, MPTCP_SUBFLOW_DATA_AVAIL); 1367 return true; 1368 } 1369 1370 bool mptcp_subflow_data_available(struct sock *sk) 1371 { 1372 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(sk); 1373 1374 /* check if current mapping is still valid */ 1375 if (subflow->map_valid && 1376 mptcp_subflow_get_map_offset(subflow) >= subflow->map_data_len) { 1377 subflow->map_valid = 0; 1378 WRITE_ONCE(subflow->data_avail, MPTCP_SUBFLOW_NODATA); 1379 1380 pr_debug("Done with mapping: seq=%u data_len=%u", 1381 subflow->map_subflow_seq, 1382 subflow->map_data_len); 1383 } 1384 1385 return subflow_check_data_avail(sk); 1386 } 1387 1388 /* If ssk has an mptcp parent socket, use the mptcp rcvbuf occupancy, 1389 * not the ssk one. 1390 * 1391 * In mptcp, rwin is about the mptcp-level connection data. 1392 * 1393 * Data that is still on the ssk rx queue can thus be ignored, 1394 * as far as mptcp peer is concerned that data is still inflight. 1395 * DSS ACK is updated when skb is moved to the mptcp rx queue. 1396 */ 1397 void mptcp_space(const struct sock *ssk, int *space, int *full_space) 1398 { 1399 const struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk); 1400 const struct sock *sk = subflow->conn; 1401 1402 *space = __mptcp_space(sk); 1403 *full_space = mptcp_win_from_space(sk, READ_ONCE(sk->sk_rcvbuf)); 1404 } 1405 1406 static void subflow_error_report(struct sock *ssk) 1407 { 1408 struct sock *sk = mptcp_subflow_ctx(ssk)->conn; 1409 1410 /* bail early if this is a no-op, so that we avoid introducing a 1411 * problematic lockdep dependency between TCP accept queue lock 1412 * and msk socket spinlock 1413 */ 1414 if (!sk->sk_socket) 1415 return; 1416 1417 mptcp_data_lock(sk); 1418 if (!sock_owned_by_user(sk)) 1419 __mptcp_error_report(sk); 1420 else 1421 __set_bit(MPTCP_ERROR_REPORT, &mptcp_sk(sk)->cb_flags); 1422 mptcp_data_unlock(sk); 1423 } 1424 1425 static void subflow_data_ready(struct sock *sk) 1426 { 1427 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(sk); 1428 u16 state = 1 << inet_sk_state_load(sk); 1429 struct sock *parent = subflow->conn; 1430 struct mptcp_sock *msk; 1431 1432 trace_sk_data_ready(sk); 1433 1434 msk = mptcp_sk(parent); 1435 if (state & TCPF_LISTEN) { 1436 /* MPJ subflow are removed from accept queue before reaching here, 1437 * avoid stray wakeups 1438 */ 1439 if (reqsk_queue_empty(&inet_csk(sk)->icsk_accept_queue)) 1440 return; 1441 1442 parent->sk_data_ready(parent); 1443 return; 1444 } 1445 1446 WARN_ON_ONCE(!__mptcp_check_fallback(msk) && !subflow->mp_capable && 1447 !subflow->mp_join && !(state & TCPF_CLOSE)); 1448 1449 if (mptcp_subflow_data_available(sk)) { 1450 mptcp_data_ready(parent, sk); 1451 1452 /* subflow-level lowat test are not relevant. 1453 * respect the msk-level threshold eventually mandating an immediate ack 1454 */ 1455 if (mptcp_data_avail(msk) < parent->sk_rcvlowat && 1456 (tcp_sk(sk)->rcv_nxt - tcp_sk(sk)->rcv_wup) > inet_csk(sk)->icsk_ack.rcv_mss) 1457 inet_csk(sk)->icsk_ack.pending |= ICSK_ACK_NOW; 1458 } else if (unlikely(sk->sk_err)) { 1459 subflow_error_report(sk); 1460 } 1461 } 1462 1463 static void subflow_write_space(struct sock *ssk) 1464 { 1465 struct sock *sk = mptcp_subflow_ctx(ssk)->conn; 1466 1467 mptcp_propagate_sndbuf(sk, ssk); 1468 mptcp_write_space(sk); 1469 } 1470 1471 static const struct inet_connection_sock_af_ops * 1472 subflow_default_af_ops(struct sock *sk) 1473 { 1474 #if IS_ENABLED(CONFIG_MPTCP_IPV6) 1475 if (sk->sk_family == AF_INET6) 1476 return &subflow_v6_specific; 1477 #endif 1478 return &subflow_specific; 1479 } 1480 1481 #if IS_ENABLED(CONFIG_MPTCP_IPV6) 1482 void mptcpv6_handle_mapped(struct sock *sk, bool mapped) 1483 { 1484 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(sk); 1485 struct inet_connection_sock *icsk = inet_csk(sk); 1486 const struct inet_connection_sock_af_ops *target; 1487 1488 target = mapped ? &subflow_v6m_specific : subflow_default_af_ops(sk); 1489 1490 pr_debug("subflow=%p family=%d ops=%p target=%p mapped=%d", 1491 subflow, sk->sk_family, icsk->icsk_af_ops, target, mapped); 1492 1493 if (likely(icsk->icsk_af_ops == target)) 1494 return; 1495 1496 subflow->icsk_af_ops = icsk->icsk_af_ops; 1497 icsk->icsk_af_ops = target; 1498 } 1499 #endif 1500 1501 void mptcp_info2sockaddr(const struct mptcp_addr_info *info, 1502 struct sockaddr_storage *addr, 1503 unsigned short family) 1504 { 1505 memset(addr, 0, sizeof(*addr)); 1506 addr->ss_family = family; 1507 if (addr->ss_family == AF_INET) { 1508 struct sockaddr_in *in_addr = (struct sockaddr_in *)addr; 1509 1510 if (info->family == AF_INET) 1511 in_addr->sin_addr = info->addr; 1512 #if IS_ENABLED(CONFIG_MPTCP_IPV6) 1513 else if (ipv6_addr_v4mapped(&info->addr6)) 1514 in_addr->sin_addr.s_addr = info->addr6.s6_addr32[3]; 1515 #endif 1516 in_addr->sin_port = info->port; 1517 } 1518 #if IS_ENABLED(CONFIG_MPTCP_IPV6) 1519 else if (addr->ss_family == AF_INET6) { 1520 struct sockaddr_in6 *in6_addr = (struct sockaddr_in6 *)addr; 1521 1522 if (info->family == AF_INET) 1523 ipv6_addr_set_v4mapped(info->addr.s_addr, 1524 &in6_addr->sin6_addr); 1525 else 1526 in6_addr->sin6_addr = info->addr6; 1527 in6_addr->sin6_port = info->port; 1528 } 1529 #endif 1530 } 1531 1532 int __mptcp_subflow_connect(struct sock *sk, const struct mptcp_addr_info *loc, 1533 const struct mptcp_addr_info *remote) 1534 { 1535 struct mptcp_sock *msk = mptcp_sk(sk); 1536 struct mptcp_subflow_context *subflow; 1537 struct sockaddr_storage addr; 1538 int remote_id = remote->id; 1539 int local_id = loc->id; 1540 int err = -ENOTCONN; 1541 struct socket *sf; 1542 struct sock *ssk; 1543 u32 remote_token; 1544 int addrlen; 1545 int ifindex; 1546 u8 flags; 1547 1548 if (!mptcp_is_fully_established(sk)) 1549 goto err_out; 1550 1551 err = mptcp_subflow_create_socket(sk, loc->family, &sf); 1552 if (err) 1553 goto err_out; 1554 1555 ssk = sf->sk; 1556 subflow = mptcp_subflow_ctx(ssk); 1557 do { 1558 get_random_bytes(&subflow->local_nonce, sizeof(u32)); 1559 } while (!subflow->local_nonce); 1560 1561 if (local_id) 1562 subflow_set_local_id(subflow, local_id); 1563 1564 mptcp_pm_get_flags_and_ifindex_by_id(msk, local_id, 1565 &flags, &ifindex); 1566 subflow->remote_key_valid = 1; 1567 subflow->remote_key = msk->remote_key; 1568 subflow->local_key = msk->local_key; 1569 subflow->token = msk->token; 1570 mptcp_info2sockaddr(loc, &addr, ssk->sk_family); 1571 1572 addrlen = sizeof(struct sockaddr_in); 1573 #if IS_ENABLED(CONFIG_MPTCP_IPV6) 1574 if (addr.ss_family == AF_INET6) 1575 addrlen = sizeof(struct sockaddr_in6); 1576 #endif 1577 mptcp_sockopt_sync(msk, ssk); 1578 1579 ssk->sk_bound_dev_if = ifindex; 1580 err = kernel_bind(sf, (struct sockaddr *)&addr, addrlen); 1581 if (err) 1582 goto failed; 1583 1584 mptcp_crypto_key_sha(subflow->remote_key, &remote_token, NULL); 1585 pr_debug("msk=%p remote_token=%u local_id=%d remote_id=%d", msk, 1586 remote_token, local_id, remote_id); 1587 subflow->remote_token = remote_token; 1588 WRITE_ONCE(subflow->remote_id, remote_id); 1589 subflow->request_join = 1; 1590 subflow->request_bkup = !!(flags & MPTCP_PM_ADDR_FLAG_BACKUP); 1591 subflow->subflow_id = msk->subflow_id++; 1592 mptcp_info2sockaddr(remote, &addr, ssk->sk_family); 1593 1594 sock_hold(ssk); 1595 list_add_tail(&subflow->node, &msk->conn_list); 1596 err = kernel_connect(sf, (struct sockaddr *)&addr, addrlen, O_NONBLOCK); 1597 if (err && err != -EINPROGRESS) 1598 goto failed_unlink; 1599 1600 /* discard the subflow socket */ 1601 mptcp_sock_graft(ssk, sk->sk_socket); 1602 iput(SOCK_INODE(sf)); 1603 WRITE_ONCE(msk->allow_infinite_fallback, false); 1604 mptcp_stop_tout_timer(sk); 1605 return 0; 1606 1607 failed_unlink: 1608 list_del(&subflow->node); 1609 sock_put(mptcp_subflow_tcp_sock(subflow)); 1610 1611 failed: 1612 subflow->disposable = 1; 1613 sock_release(sf); 1614 1615 err_out: 1616 /* we account subflows before the creation, and this failures will not 1617 * be caught by sk_state_change() 1618 */ 1619 mptcp_pm_close_subflow(msk); 1620 return err; 1621 } 1622 1623 static void mptcp_attach_cgroup(struct sock *parent, struct sock *child) 1624 { 1625 #ifdef CONFIG_SOCK_CGROUP_DATA 1626 struct sock_cgroup_data *parent_skcd = &parent->sk_cgrp_data, 1627 *child_skcd = &child->sk_cgrp_data; 1628 1629 /* only the additional subflows created by kworkers have to be modified */ 1630 if (cgroup_id(sock_cgroup_ptr(parent_skcd)) != 1631 cgroup_id(sock_cgroup_ptr(child_skcd))) { 1632 #ifdef CONFIG_MEMCG 1633 struct mem_cgroup *memcg = parent->sk_memcg; 1634 1635 mem_cgroup_sk_free(child); 1636 if (memcg && css_tryget(&memcg->css)) 1637 child->sk_memcg = memcg; 1638 #endif /* CONFIG_MEMCG */ 1639 1640 cgroup_sk_free(child_skcd); 1641 *child_skcd = *parent_skcd; 1642 cgroup_sk_clone(child_skcd); 1643 } 1644 #endif /* CONFIG_SOCK_CGROUP_DATA */ 1645 } 1646 1647 static void mptcp_subflow_ops_override(struct sock *ssk) 1648 { 1649 #if IS_ENABLED(CONFIG_MPTCP_IPV6) 1650 if (ssk->sk_prot == &tcpv6_prot) 1651 ssk->sk_prot = &tcpv6_prot_override; 1652 else 1653 #endif 1654 ssk->sk_prot = &tcp_prot_override; 1655 } 1656 1657 static void mptcp_subflow_ops_undo_override(struct sock *ssk) 1658 { 1659 #if IS_ENABLED(CONFIG_MPTCP_IPV6) 1660 if (ssk->sk_prot == &tcpv6_prot_override) 1661 ssk->sk_prot = &tcpv6_prot; 1662 else 1663 #endif 1664 ssk->sk_prot = &tcp_prot; 1665 } 1666 1667 int mptcp_subflow_create_socket(struct sock *sk, unsigned short family, 1668 struct socket **new_sock) 1669 { 1670 struct mptcp_subflow_context *subflow; 1671 struct net *net = sock_net(sk); 1672 struct socket *sf; 1673 int err; 1674 1675 /* un-accepted server sockets can reach here - on bad configuration 1676 * bail early to avoid greater trouble later 1677 */ 1678 if (unlikely(!sk->sk_socket)) 1679 return -EINVAL; 1680 1681 err = sock_create_kern(net, family, SOCK_STREAM, IPPROTO_TCP, &sf); 1682 if (err) 1683 return err; 1684 1685 lock_sock_nested(sf->sk, SINGLE_DEPTH_NESTING); 1686 1687 err = security_mptcp_add_subflow(sk, sf->sk); 1688 if (err) 1689 goto release_ssk; 1690 1691 /* the newly created socket has to be in the same cgroup as its parent */ 1692 mptcp_attach_cgroup(sk, sf->sk); 1693 1694 /* kernel sockets do not by default acquire net ref, but TCP timer 1695 * needs it. 1696 * Update ns_tracker to current stack trace and refcounted tracker. 1697 */ 1698 __netns_tracker_free(net, &sf->sk->ns_tracker, false); 1699 sf->sk->sk_net_refcnt = 1; 1700 get_net_track(net, &sf->sk->ns_tracker, GFP_KERNEL); 1701 sock_inuse_add(net, 1); 1702 err = tcp_set_ulp(sf->sk, "mptcp"); 1703 1704 release_ssk: 1705 release_sock(sf->sk); 1706 1707 if (err) { 1708 sock_release(sf); 1709 return err; 1710 } 1711 1712 /* the newly created socket really belongs to the owning MPTCP master 1713 * socket, even if for additional subflows the allocation is performed 1714 * by a kernel workqueue. Adjust inode references, so that the 1715 * procfs/diag interfaces really show this one belonging to the correct 1716 * user. 1717 */ 1718 SOCK_INODE(sf)->i_ino = SOCK_INODE(sk->sk_socket)->i_ino; 1719 SOCK_INODE(sf)->i_uid = SOCK_INODE(sk->sk_socket)->i_uid; 1720 SOCK_INODE(sf)->i_gid = SOCK_INODE(sk->sk_socket)->i_gid; 1721 1722 subflow = mptcp_subflow_ctx(sf->sk); 1723 pr_debug("subflow=%p", subflow); 1724 1725 *new_sock = sf; 1726 sock_hold(sk); 1727 subflow->conn = sk; 1728 mptcp_subflow_ops_override(sf->sk); 1729 1730 return 0; 1731 } 1732 1733 static struct mptcp_subflow_context *subflow_create_ctx(struct sock *sk, 1734 gfp_t priority) 1735 { 1736 struct inet_connection_sock *icsk = inet_csk(sk); 1737 struct mptcp_subflow_context *ctx; 1738 1739 ctx = kzalloc(sizeof(*ctx), priority); 1740 if (!ctx) 1741 return NULL; 1742 1743 rcu_assign_pointer(icsk->icsk_ulp_data, ctx); 1744 INIT_LIST_HEAD(&ctx->node); 1745 INIT_LIST_HEAD(&ctx->delegated_node); 1746 1747 pr_debug("subflow=%p", ctx); 1748 1749 ctx->tcp_sock = sk; 1750 WRITE_ONCE(ctx->local_id, -1); 1751 1752 return ctx; 1753 } 1754 1755 static void __subflow_state_change(struct sock *sk) 1756 { 1757 struct socket_wq *wq; 1758 1759 rcu_read_lock(); 1760 wq = rcu_dereference(sk->sk_wq); 1761 if (skwq_has_sleeper(wq)) 1762 wake_up_interruptible_all(&wq->wait); 1763 rcu_read_unlock(); 1764 } 1765 1766 static bool subflow_is_done(const struct sock *sk) 1767 { 1768 return sk->sk_shutdown & RCV_SHUTDOWN || sk->sk_state == TCP_CLOSE; 1769 } 1770 1771 static void subflow_state_change(struct sock *sk) 1772 { 1773 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(sk); 1774 struct sock *parent = subflow->conn; 1775 struct mptcp_sock *msk; 1776 1777 __subflow_state_change(sk); 1778 1779 msk = mptcp_sk(parent); 1780 if (subflow_simultaneous_connect(sk)) { 1781 mptcp_do_fallback(sk); 1782 pr_fallback(msk); 1783 subflow->conn_finished = 1; 1784 mptcp_propagate_state(parent, sk, subflow, NULL); 1785 } 1786 1787 /* as recvmsg() does not acquire the subflow socket for ssk selection 1788 * a fin packet carrying a DSS can be unnoticed if we don't trigger 1789 * the data available machinery here. 1790 */ 1791 if (mptcp_subflow_data_available(sk)) 1792 mptcp_data_ready(parent, sk); 1793 else if (unlikely(sk->sk_err)) 1794 subflow_error_report(sk); 1795 1796 subflow_sched_work_if_closed(mptcp_sk(parent), sk); 1797 1798 /* when the fallback subflow closes the rx side, trigger a 'dummy' 1799 * ingress data fin, so that the msk state will follow along 1800 */ 1801 if (__mptcp_check_fallback(msk) && subflow_is_done(sk) && msk->first == sk && 1802 mptcp_update_rcv_data_fin(msk, READ_ONCE(msk->ack_seq), true)) 1803 mptcp_schedule_work(parent); 1804 } 1805 1806 void mptcp_subflow_queue_clean(struct sock *listener_sk, struct sock *listener_ssk) 1807 { 1808 struct request_sock_queue *queue = &inet_csk(listener_ssk)->icsk_accept_queue; 1809 struct request_sock *req, *head, *tail; 1810 struct mptcp_subflow_context *subflow; 1811 struct sock *sk, *ssk; 1812 1813 /* Due to lock dependencies no relevant lock can be acquired under rskq_lock. 1814 * Splice the req list, so that accept() can not reach the pending ssk after 1815 * the listener socket is released below. 1816 */ 1817 spin_lock_bh(&queue->rskq_lock); 1818 head = queue->rskq_accept_head; 1819 tail = queue->rskq_accept_tail; 1820 queue->rskq_accept_head = NULL; 1821 queue->rskq_accept_tail = NULL; 1822 spin_unlock_bh(&queue->rskq_lock); 1823 1824 if (!head) 1825 return; 1826 1827 /* can't acquire the msk socket lock under the subflow one, 1828 * or will cause ABBA deadlock 1829 */ 1830 release_sock(listener_ssk); 1831 1832 for (req = head; req; req = req->dl_next) { 1833 ssk = req->sk; 1834 if (!sk_is_mptcp(ssk)) 1835 continue; 1836 1837 subflow = mptcp_subflow_ctx(ssk); 1838 if (!subflow || !subflow->conn) 1839 continue; 1840 1841 sk = subflow->conn; 1842 sock_hold(sk); 1843 1844 lock_sock_nested(sk, SINGLE_DEPTH_NESTING); 1845 __mptcp_unaccepted_force_close(sk); 1846 release_sock(sk); 1847 1848 /* lockdep will report a false positive ABBA deadlock 1849 * between cancel_work_sync and the listener socket. 1850 * The involved locks belong to different sockets WRT 1851 * the existing AB chain. 1852 * Using a per socket key is problematic as key 1853 * deregistration requires process context and must be 1854 * performed at socket disposal time, in atomic 1855 * context. 1856 * Just tell lockdep to consider the listener socket 1857 * released here. 1858 */ 1859 mutex_release(&listener_sk->sk_lock.dep_map, _RET_IP_); 1860 mptcp_cancel_work(sk); 1861 mutex_acquire(&listener_sk->sk_lock.dep_map, 0, 0, _RET_IP_); 1862 1863 sock_put(sk); 1864 } 1865 1866 /* we are still under the listener msk socket lock */ 1867 lock_sock_nested(listener_ssk, SINGLE_DEPTH_NESTING); 1868 1869 /* restore the listener queue, to let the TCP code clean it up */ 1870 spin_lock_bh(&queue->rskq_lock); 1871 WARN_ON_ONCE(queue->rskq_accept_head); 1872 queue->rskq_accept_head = head; 1873 queue->rskq_accept_tail = tail; 1874 spin_unlock_bh(&queue->rskq_lock); 1875 } 1876 1877 static int subflow_ulp_init(struct sock *sk) 1878 { 1879 struct inet_connection_sock *icsk = inet_csk(sk); 1880 struct mptcp_subflow_context *ctx; 1881 struct tcp_sock *tp = tcp_sk(sk); 1882 int err = 0; 1883 1884 /* disallow attaching ULP to a socket unless it has been 1885 * created with sock_create_kern() 1886 */ 1887 if (!sk->sk_kern_sock) { 1888 err = -EOPNOTSUPP; 1889 goto out; 1890 } 1891 1892 ctx = subflow_create_ctx(sk, GFP_KERNEL); 1893 if (!ctx) { 1894 err = -ENOMEM; 1895 goto out; 1896 } 1897 1898 pr_debug("subflow=%p, family=%d", ctx, sk->sk_family); 1899 1900 tp->is_mptcp = 1; 1901 ctx->icsk_af_ops = icsk->icsk_af_ops; 1902 icsk->icsk_af_ops = subflow_default_af_ops(sk); 1903 ctx->tcp_state_change = sk->sk_state_change; 1904 ctx->tcp_error_report = sk->sk_error_report; 1905 1906 WARN_ON_ONCE(sk->sk_data_ready != sock_def_readable); 1907 WARN_ON_ONCE(sk->sk_write_space != sk_stream_write_space); 1908 1909 sk->sk_data_ready = subflow_data_ready; 1910 sk->sk_write_space = subflow_write_space; 1911 sk->sk_state_change = subflow_state_change; 1912 sk->sk_error_report = subflow_error_report; 1913 out: 1914 return err; 1915 } 1916 1917 static void subflow_ulp_release(struct sock *ssk) 1918 { 1919 struct mptcp_subflow_context *ctx = mptcp_subflow_ctx(ssk); 1920 bool release = true; 1921 struct sock *sk; 1922 1923 if (!ctx) 1924 return; 1925 1926 sk = ctx->conn; 1927 if (sk) { 1928 /* if the msk has been orphaned, keep the ctx 1929 * alive, will be freed by __mptcp_close_ssk(), 1930 * when the subflow is still unaccepted 1931 */ 1932 release = ctx->disposable || list_empty(&ctx->node); 1933 1934 /* inet_child_forget() does not call sk_state_change(), 1935 * explicitly trigger the socket close machinery 1936 */ 1937 if (!release && !test_and_set_bit(MPTCP_WORK_CLOSE_SUBFLOW, 1938 &mptcp_sk(sk)->flags)) 1939 mptcp_schedule_work(sk); 1940 sock_put(sk); 1941 } 1942 1943 mptcp_subflow_ops_undo_override(ssk); 1944 if (release) 1945 kfree_rcu(ctx, rcu); 1946 } 1947 1948 static void subflow_ulp_clone(const struct request_sock *req, 1949 struct sock *newsk, 1950 const gfp_t priority) 1951 { 1952 struct mptcp_subflow_request_sock *subflow_req = mptcp_subflow_rsk(req); 1953 struct mptcp_subflow_context *old_ctx = mptcp_subflow_ctx(newsk); 1954 struct mptcp_subflow_context *new_ctx; 1955 1956 if (!tcp_rsk(req)->is_mptcp || 1957 (!subflow_req->mp_capable && !subflow_req->mp_join)) { 1958 subflow_ulp_fallback(newsk, old_ctx); 1959 return; 1960 } 1961 1962 new_ctx = subflow_create_ctx(newsk, priority); 1963 if (!new_ctx) { 1964 subflow_ulp_fallback(newsk, old_ctx); 1965 return; 1966 } 1967 1968 new_ctx->conn_finished = 1; 1969 new_ctx->icsk_af_ops = old_ctx->icsk_af_ops; 1970 new_ctx->tcp_state_change = old_ctx->tcp_state_change; 1971 new_ctx->tcp_error_report = old_ctx->tcp_error_report; 1972 new_ctx->rel_write_seq = 1; 1973 new_ctx->tcp_sock = newsk; 1974 1975 if (subflow_req->mp_capable) { 1976 /* see comments in subflow_syn_recv_sock(), MPTCP connection 1977 * is fully established only after we receive the remote key 1978 */ 1979 new_ctx->mp_capable = 1; 1980 new_ctx->local_key = subflow_req->local_key; 1981 new_ctx->token = subflow_req->token; 1982 new_ctx->ssn_offset = subflow_req->ssn_offset; 1983 new_ctx->idsn = subflow_req->idsn; 1984 1985 /* this is the first subflow, id is always 0 */ 1986 subflow_set_local_id(new_ctx, 0); 1987 } else if (subflow_req->mp_join) { 1988 new_ctx->ssn_offset = subflow_req->ssn_offset; 1989 new_ctx->mp_join = 1; 1990 new_ctx->fully_established = 1; 1991 new_ctx->remote_key_valid = 1; 1992 new_ctx->backup = subflow_req->backup; 1993 new_ctx->request_bkup = subflow_req->request_bkup; 1994 WRITE_ONCE(new_ctx->remote_id, subflow_req->remote_id); 1995 new_ctx->token = subflow_req->token; 1996 new_ctx->thmac = subflow_req->thmac; 1997 1998 /* the subflow req id is valid, fetched via subflow_check_req() 1999 * and subflow_token_join_request() 2000 */ 2001 subflow_set_local_id(new_ctx, subflow_req->local_id); 2002 } 2003 } 2004 2005 static void tcp_release_cb_override(struct sock *ssk) 2006 { 2007 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk); 2008 long status; 2009 2010 /* process and clear all the pending actions, but leave the subflow into 2011 * the napi queue. To respect locking, only the same CPU that originated 2012 * the action can touch the list. mptcp_napi_poll will take care of it. 2013 */ 2014 status = set_mask_bits(&subflow->delegated_status, MPTCP_DELEGATE_ACTIONS_MASK, 0); 2015 if (status) 2016 mptcp_subflow_process_delegated(ssk, status); 2017 2018 tcp_release_cb(ssk); 2019 } 2020 2021 static int tcp_abort_override(struct sock *ssk, int err) 2022 { 2023 /* closing a listener subflow requires a great deal of care. 2024 * keep it simple and just prevent such operation 2025 */ 2026 if (inet_sk_state_load(ssk) == TCP_LISTEN) 2027 return -EINVAL; 2028 2029 return tcp_abort(ssk, err); 2030 } 2031 2032 static struct tcp_ulp_ops subflow_ulp_ops __read_mostly = { 2033 .name = "mptcp", 2034 .owner = THIS_MODULE, 2035 .init = subflow_ulp_init, 2036 .release = subflow_ulp_release, 2037 .clone = subflow_ulp_clone, 2038 }; 2039 2040 static int subflow_ops_init(struct request_sock_ops *subflow_ops) 2041 { 2042 subflow_ops->obj_size = sizeof(struct mptcp_subflow_request_sock); 2043 2044 subflow_ops->slab = kmem_cache_create(subflow_ops->slab_name, 2045 subflow_ops->obj_size, 0, 2046 SLAB_ACCOUNT | 2047 SLAB_TYPESAFE_BY_RCU, 2048 NULL); 2049 if (!subflow_ops->slab) 2050 return -ENOMEM; 2051 2052 return 0; 2053 } 2054 2055 void __init mptcp_subflow_init(void) 2056 { 2057 mptcp_subflow_v4_request_sock_ops = tcp_request_sock_ops; 2058 mptcp_subflow_v4_request_sock_ops.slab_name = "request_sock_subflow_v4"; 2059 mptcp_subflow_v4_request_sock_ops.destructor = subflow_v4_req_destructor; 2060 2061 if (subflow_ops_init(&mptcp_subflow_v4_request_sock_ops) != 0) 2062 panic("MPTCP: failed to init subflow v4 request sock ops\n"); 2063 2064 subflow_request_sock_ipv4_ops = tcp_request_sock_ipv4_ops; 2065 subflow_request_sock_ipv4_ops.route_req = subflow_v4_route_req; 2066 subflow_request_sock_ipv4_ops.send_synack = subflow_v4_send_synack; 2067 2068 subflow_specific = ipv4_specific; 2069 subflow_specific.conn_request = subflow_v4_conn_request; 2070 subflow_specific.syn_recv_sock = subflow_syn_recv_sock; 2071 subflow_specific.sk_rx_dst_set = subflow_finish_connect; 2072 subflow_specific.rebuild_header = subflow_rebuild_header; 2073 2074 tcp_prot_override = tcp_prot; 2075 tcp_prot_override.release_cb = tcp_release_cb_override; 2076 tcp_prot_override.diag_destroy = tcp_abort_override; 2077 2078 #if IS_ENABLED(CONFIG_MPTCP_IPV6) 2079 /* In struct mptcp_subflow_request_sock, we assume the TCP request sock 2080 * structures for v4 and v6 have the same size. It should not changed in 2081 * the future but better to make sure to be warned if it is no longer 2082 * the case. 2083 */ 2084 BUILD_BUG_ON(sizeof(struct tcp_request_sock) != sizeof(struct tcp6_request_sock)); 2085 2086 mptcp_subflow_v6_request_sock_ops = tcp6_request_sock_ops; 2087 mptcp_subflow_v6_request_sock_ops.slab_name = "request_sock_subflow_v6"; 2088 mptcp_subflow_v6_request_sock_ops.destructor = subflow_v6_req_destructor; 2089 2090 if (subflow_ops_init(&mptcp_subflow_v6_request_sock_ops) != 0) 2091 panic("MPTCP: failed to init subflow v6 request sock ops\n"); 2092 2093 subflow_request_sock_ipv6_ops = tcp_request_sock_ipv6_ops; 2094 subflow_request_sock_ipv6_ops.route_req = subflow_v6_route_req; 2095 subflow_request_sock_ipv6_ops.send_synack = subflow_v6_send_synack; 2096 2097 subflow_v6_specific = ipv6_specific; 2098 subflow_v6_specific.conn_request = subflow_v6_conn_request; 2099 subflow_v6_specific.syn_recv_sock = subflow_syn_recv_sock; 2100 subflow_v6_specific.sk_rx_dst_set = subflow_finish_connect; 2101 subflow_v6_specific.rebuild_header = subflow_v6_rebuild_header; 2102 2103 subflow_v6m_specific = subflow_v6_specific; 2104 subflow_v6m_specific.queue_xmit = ipv4_specific.queue_xmit; 2105 subflow_v6m_specific.send_check = ipv4_specific.send_check; 2106 subflow_v6m_specific.net_header_len = ipv4_specific.net_header_len; 2107 subflow_v6m_specific.mtu_reduced = ipv4_specific.mtu_reduced; 2108 subflow_v6m_specific.net_frag_header_len = 0; 2109 subflow_v6m_specific.rebuild_header = subflow_rebuild_header; 2110 2111 tcpv6_prot_override = tcpv6_prot; 2112 tcpv6_prot_override.release_cb = tcp_release_cb_override; 2113 tcpv6_prot_override.diag_destroy = tcp_abort_override; 2114 #endif 2115 2116 mptcp_diag_subflow_init(&subflow_ulp_ops); 2117 2118 if (tcp_register_ulp(&subflow_ulp_ops) != 0) 2119 panic("MPTCP: failed to register subflows to ULP\n"); 2120 } 2121