1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * INET An implementation of the TCP/IP protocol suite for the LINUX 4 * operating system. INET is implemented using the BSD Socket 5 * interface as the means of communication with the user level. 6 * 7 * Support for INET connection oriented protocols. 8 * 9 * Authors: See the TCP sources 10 */ 11 12 #include <linux/module.h> 13 #include <linux/jhash.h> 14 15 #include <net/inet_connection_sock.h> 16 #include <net/inet_hashtables.h> 17 #include <net/inet_timewait_sock.h> 18 #include <net/ip.h> 19 #include <net/route.h> 20 #include <net/tcp_states.h> 21 #include <net/xfrm.h> 22 #include <net/tcp.h> 23 #include <net/sock_reuseport.h> 24 #include <net/addrconf.h> 25 26 #if IS_ENABLED(CONFIG_IPV6) 27 /* match_sk*_wildcard == true: IPV6_ADDR_ANY equals to any IPv6 addresses 28 * if IPv6 only, and any IPv4 addresses 29 * if not IPv6 only 30 * match_sk*_wildcard == false: addresses must be exactly the same, i.e. 31 * IPV6_ADDR_ANY only equals to IPV6_ADDR_ANY, 32 * and 0.0.0.0 equals to 0.0.0.0 only 33 */ 34 static bool ipv6_rcv_saddr_equal(const struct in6_addr *sk1_rcv_saddr6, 35 const struct in6_addr *sk2_rcv_saddr6, 36 __be32 sk1_rcv_saddr, __be32 sk2_rcv_saddr, 37 bool sk1_ipv6only, bool sk2_ipv6only, 38 bool match_sk1_wildcard, 39 bool match_sk2_wildcard) 40 { 41 int addr_type = ipv6_addr_type(sk1_rcv_saddr6); 42 int addr_type2 = sk2_rcv_saddr6 ? ipv6_addr_type(sk2_rcv_saddr6) : IPV6_ADDR_MAPPED; 43 44 /* if both are mapped, treat as IPv4 */ 45 if (addr_type == IPV6_ADDR_MAPPED && addr_type2 == IPV6_ADDR_MAPPED) { 46 if (!sk2_ipv6only) { 47 if (sk1_rcv_saddr == sk2_rcv_saddr) 48 return true; 49 return (match_sk1_wildcard && !sk1_rcv_saddr) || 50 (match_sk2_wildcard && !sk2_rcv_saddr); 51 } 52 return false; 53 } 54 55 if (addr_type == IPV6_ADDR_ANY && addr_type2 == IPV6_ADDR_ANY) 56 return true; 57 58 if (addr_type2 == IPV6_ADDR_ANY && match_sk2_wildcard && 59 !(sk2_ipv6only && addr_type == IPV6_ADDR_MAPPED)) 60 return true; 61 62 if (addr_type == IPV6_ADDR_ANY && match_sk1_wildcard && 63 !(sk1_ipv6only && addr_type2 == IPV6_ADDR_MAPPED)) 64 return true; 65 66 if (sk2_rcv_saddr6 && 67 ipv6_addr_equal(sk1_rcv_saddr6, sk2_rcv_saddr6)) 68 return true; 69 70 return false; 71 } 72 #endif 73 74 /* match_sk*_wildcard == true: 0.0.0.0 equals to any IPv4 addresses 75 * match_sk*_wildcard == false: addresses must be exactly the same, i.e. 76 * 0.0.0.0 only equals to 0.0.0.0 77 */ 78 static bool ipv4_rcv_saddr_equal(__be32 sk1_rcv_saddr, __be32 sk2_rcv_saddr, 79 bool sk2_ipv6only, bool match_sk1_wildcard, 80 bool match_sk2_wildcard) 81 { 82 if (!sk2_ipv6only) { 83 if (sk1_rcv_saddr == sk2_rcv_saddr) 84 return true; 85 return (match_sk1_wildcard && !sk1_rcv_saddr) || 86 (match_sk2_wildcard && !sk2_rcv_saddr); 87 } 88 return false; 89 } 90 91 bool inet_rcv_saddr_equal(const struct sock *sk, const struct sock *sk2, 92 bool match_wildcard) 93 { 94 #if IS_ENABLED(CONFIG_IPV6) 95 if (sk->sk_family == AF_INET6) 96 return ipv6_rcv_saddr_equal(&sk->sk_v6_rcv_saddr, 97 inet6_rcv_saddr(sk2), 98 sk->sk_rcv_saddr, 99 sk2->sk_rcv_saddr, 100 ipv6_only_sock(sk), 101 ipv6_only_sock(sk2), 102 match_wildcard, 103 match_wildcard); 104 #endif 105 return ipv4_rcv_saddr_equal(sk->sk_rcv_saddr, sk2->sk_rcv_saddr, 106 ipv6_only_sock(sk2), match_wildcard, 107 match_wildcard); 108 } 109 EXPORT_SYMBOL(inet_rcv_saddr_equal); 110 111 bool inet_rcv_saddr_any(const struct sock *sk) 112 { 113 #if IS_ENABLED(CONFIG_IPV6) 114 if (sk->sk_family == AF_INET6) 115 return ipv6_addr_any(&sk->sk_v6_rcv_saddr); 116 #endif 117 return !sk->sk_rcv_saddr; 118 } 119 120 void inet_get_local_port_range(struct net *net, int *low, int *high) 121 { 122 unsigned int seq; 123 124 do { 125 seq = read_seqbegin(&net->ipv4.ip_local_ports.lock); 126 127 *low = net->ipv4.ip_local_ports.range[0]; 128 *high = net->ipv4.ip_local_ports.range[1]; 129 } while (read_seqretry(&net->ipv4.ip_local_ports.lock, seq)); 130 } 131 EXPORT_SYMBOL(inet_get_local_port_range); 132 133 static bool inet_use_bhash2_on_bind(const struct sock *sk) 134 { 135 #if IS_ENABLED(CONFIG_IPV6) 136 if (sk->sk_family == AF_INET6) { 137 int addr_type = ipv6_addr_type(&sk->sk_v6_rcv_saddr); 138 139 return addr_type != IPV6_ADDR_ANY && 140 addr_type != IPV6_ADDR_MAPPED; 141 } 142 #endif 143 return sk->sk_rcv_saddr != htonl(INADDR_ANY); 144 } 145 146 static bool inet_bind_conflict(const struct sock *sk, struct sock *sk2, 147 kuid_t sk_uid, bool relax, 148 bool reuseport_cb_ok, bool reuseport_ok) 149 { 150 int bound_dev_if2; 151 152 if (sk == sk2) 153 return false; 154 155 bound_dev_if2 = READ_ONCE(sk2->sk_bound_dev_if); 156 157 if (!sk->sk_bound_dev_if || !bound_dev_if2 || 158 sk->sk_bound_dev_if == bound_dev_if2) { 159 if (sk->sk_reuse && sk2->sk_reuse && 160 sk2->sk_state != TCP_LISTEN) { 161 if (!relax || (!reuseport_ok && sk->sk_reuseport && 162 sk2->sk_reuseport && reuseport_cb_ok && 163 (sk2->sk_state == TCP_TIME_WAIT || 164 uid_eq(sk_uid, sock_i_uid(sk2))))) 165 return true; 166 } else if (!reuseport_ok || !sk->sk_reuseport || 167 !sk2->sk_reuseport || !reuseport_cb_ok || 168 (sk2->sk_state != TCP_TIME_WAIT && 169 !uid_eq(sk_uid, sock_i_uid(sk2)))) { 170 return true; 171 } 172 } 173 return false; 174 } 175 176 static bool inet_bhash2_conflict(const struct sock *sk, 177 const struct inet_bind2_bucket *tb2, 178 kuid_t sk_uid, 179 bool relax, bool reuseport_cb_ok, 180 bool reuseport_ok) 181 { 182 struct sock *sk2; 183 184 sk_for_each_bound_bhash2(sk2, &tb2->owners) { 185 if (sk->sk_family == AF_INET && ipv6_only_sock(sk2)) 186 continue; 187 188 if (inet_bind_conflict(sk, sk2, sk_uid, relax, 189 reuseport_cb_ok, reuseport_ok)) 190 return true; 191 } 192 return false; 193 } 194 195 /* This should be called only when the tb and tb2 hashbuckets' locks are held */ 196 static int inet_csk_bind_conflict(const struct sock *sk, 197 const struct inet_bind_bucket *tb, 198 const struct inet_bind2_bucket *tb2, /* may be null */ 199 bool relax, bool reuseport_ok) 200 { 201 bool reuseport_cb_ok; 202 struct sock_reuseport *reuseport_cb; 203 kuid_t uid = sock_i_uid((struct sock *)sk); 204 205 rcu_read_lock(); 206 reuseport_cb = rcu_dereference(sk->sk_reuseport_cb); 207 /* paired with WRITE_ONCE() in __reuseport_(add|detach)_closed_sock */ 208 reuseport_cb_ok = !reuseport_cb || READ_ONCE(reuseport_cb->num_closed_socks); 209 rcu_read_unlock(); 210 211 /* 212 * Unlike other sk lookup places we do not check 213 * for sk_net here, since _all_ the socks listed 214 * in tb->owners and tb2->owners list belong 215 * to the same net - the one this bucket belongs to. 216 */ 217 218 if (!inet_use_bhash2_on_bind(sk)) { 219 struct sock *sk2; 220 221 sk_for_each_bound(sk2, &tb->owners) 222 if (inet_bind_conflict(sk, sk2, uid, relax, 223 reuseport_cb_ok, reuseport_ok) && 224 inet_rcv_saddr_equal(sk, sk2, true)) 225 return true; 226 227 return false; 228 } 229 230 /* Conflicts with an existing IPV6_ADDR_ANY (if ipv6) or INADDR_ANY (if 231 * ipv4) should have been checked already. We need to do these two 232 * checks separately because their spinlocks have to be acquired/released 233 * independently of each other, to prevent possible deadlocks 234 */ 235 return tb2 && inet_bhash2_conflict(sk, tb2, uid, relax, reuseport_cb_ok, 236 reuseport_ok); 237 } 238 239 /* Determine if there is a bind conflict with an existing IPV6_ADDR_ANY (if ipv6) or 240 * INADDR_ANY (if ipv4) socket. 241 * 242 * Caller must hold bhash hashbucket lock with local bh disabled, to protect 243 * against concurrent binds on the port for addr any 244 */ 245 static bool inet_bhash2_addr_any_conflict(const struct sock *sk, int port, int l3mdev, 246 bool relax, bool reuseport_ok) 247 { 248 kuid_t uid = sock_i_uid((struct sock *)sk); 249 const struct net *net = sock_net(sk); 250 struct sock_reuseport *reuseport_cb; 251 struct inet_bind_hashbucket *head2; 252 struct inet_bind2_bucket *tb2; 253 bool reuseport_cb_ok; 254 255 rcu_read_lock(); 256 reuseport_cb = rcu_dereference(sk->sk_reuseport_cb); 257 /* paired with WRITE_ONCE() in __reuseport_(add|detach)_closed_sock */ 258 reuseport_cb_ok = !reuseport_cb || READ_ONCE(reuseport_cb->num_closed_socks); 259 rcu_read_unlock(); 260 261 head2 = inet_bhash2_addr_any_hashbucket(sk, net, port); 262 263 spin_lock(&head2->lock); 264 265 inet_bind_bucket_for_each(tb2, &head2->chain) 266 if (inet_bind2_bucket_match_addr_any(tb2, net, port, l3mdev, sk)) 267 break; 268 269 if (tb2 && inet_bhash2_conflict(sk, tb2, uid, relax, reuseport_cb_ok, 270 reuseport_ok)) { 271 spin_unlock(&head2->lock); 272 return true; 273 } 274 275 spin_unlock(&head2->lock); 276 return false; 277 } 278 279 /* 280 * Find an open port number for the socket. Returns with the 281 * inet_bind_hashbucket locks held if successful. 282 */ 283 static struct inet_bind_hashbucket * 284 inet_csk_find_open_port(const struct sock *sk, struct inet_bind_bucket **tb_ret, 285 struct inet_bind2_bucket **tb2_ret, 286 struct inet_bind_hashbucket **head2_ret, int *port_ret) 287 { 288 struct inet_hashinfo *hinfo = tcp_or_dccp_get_hashinfo(sk); 289 int i, low, high, attempt_half, port, l3mdev; 290 struct inet_bind_hashbucket *head, *head2; 291 struct net *net = sock_net(sk); 292 struct inet_bind2_bucket *tb2; 293 struct inet_bind_bucket *tb; 294 u32 remaining, offset; 295 bool relax = false; 296 297 l3mdev = inet_sk_bound_l3mdev(sk); 298 ports_exhausted: 299 attempt_half = (sk->sk_reuse == SK_CAN_REUSE) ? 1 : 0; 300 other_half_scan: 301 inet_get_local_port_range(net, &low, &high); 302 high++; /* [32768, 60999] -> [32768, 61000[ */ 303 if (high - low < 4) 304 attempt_half = 0; 305 if (attempt_half) { 306 int half = low + (((high - low) >> 2) << 1); 307 308 if (attempt_half == 1) 309 high = half; 310 else 311 low = half; 312 } 313 remaining = high - low; 314 if (likely(remaining > 1)) 315 remaining &= ~1U; 316 317 offset = prandom_u32_max(remaining); 318 /* __inet_hash_connect() favors ports having @low parity 319 * We do the opposite to not pollute connect() users. 320 */ 321 offset |= 1U; 322 323 other_parity_scan: 324 port = low + offset; 325 for (i = 0; i < remaining; i += 2, port += 2) { 326 if (unlikely(port >= high)) 327 port -= remaining; 328 if (inet_is_local_reserved_port(net, port)) 329 continue; 330 head = &hinfo->bhash[inet_bhashfn(net, port, 331 hinfo->bhash_size)]; 332 spin_lock_bh(&head->lock); 333 if (inet_use_bhash2_on_bind(sk)) { 334 if (inet_bhash2_addr_any_conflict(sk, port, l3mdev, relax, false)) 335 goto next_port; 336 } 337 338 head2 = inet_bhashfn_portaddr(hinfo, sk, net, port); 339 spin_lock(&head2->lock); 340 tb2 = inet_bind2_bucket_find(head2, net, port, l3mdev, sk); 341 inet_bind_bucket_for_each(tb, &head->chain) 342 if (inet_bind_bucket_match(tb, net, port, l3mdev)) { 343 if (!inet_csk_bind_conflict(sk, tb, tb2, 344 relax, false)) 345 goto success; 346 spin_unlock(&head2->lock); 347 goto next_port; 348 } 349 tb = NULL; 350 goto success; 351 next_port: 352 spin_unlock_bh(&head->lock); 353 cond_resched(); 354 } 355 356 offset--; 357 if (!(offset & 1)) 358 goto other_parity_scan; 359 360 if (attempt_half == 1) { 361 /* OK we now try the upper half of the range */ 362 attempt_half = 2; 363 goto other_half_scan; 364 } 365 366 if (READ_ONCE(net->ipv4.sysctl_ip_autobind_reuse) && !relax) { 367 /* We still have a chance to connect to different destinations */ 368 relax = true; 369 goto ports_exhausted; 370 } 371 return NULL; 372 success: 373 *port_ret = port; 374 *tb_ret = tb; 375 *tb2_ret = tb2; 376 *head2_ret = head2; 377 return head; 378 } 379 380 static inline int sk_reuseport_match(struct inet_bind_bucket *tb, 381 struct sock *sk) 382 { 383 kuid_t uid = sock_i_uid(sk); 384 385 if (tb->fastreuseport <= 0) 386 return 0; 387 if (!sk->sk_reuseport) 388 return 0; 389 if (rcu_access_pointer(sk->sk_reuseport_cb)) 390 return 0; 391 if (!uid_eq(tb->fastuid, uid)) 392 return 0; 393 /* We only need to check the rcv_saddr if this tb was once marked 394 * without fastreuseport and then was reset, as we can only know that 395 * the fast_*rcv_saddr doesn't have any conflicts with the socks on the 396 * owners list. 397 */ 398 if (tb->fastreuseport == FASTREUSEPORT_ANY) 399 return 1; 400 #if IS_ENABLED(CONFIG_IPV6) 401 if (tb->fast_sk_family == AF_INET6) 402 return ipv6_rcv_saddr_equal(&tb->fast_v6_rcv_saddr, 403 inet6_rcv_saddr(sk), 404 tb->fast_rcv_saddr, 405 sk->sk_rcv_saddr, 406 tb->fast_ipv6_only, 407 ipv6_only_sock(sk), true, false); 408 #endif 409 return ipv4_rcv_saddr_equal(tb->fast_rcv_saddr, sk->sk_rcv_saddr, 410 ipv6_only_sock(sk), true, false); 411 } 412 413 void inet_csk_update_fastreuse(struct inet_bind_bucket *tb, 414 struct sock *sk) 415 { 416 kuid_t uid = sock_i_uid(sk); 417 bool reuse = sk->sk_reuse && sk->sk_state != TCP_LISTEN; 418 419 if (hlist_empty(&tb->owners)) { 420 tb->fastreuse = reuse; 421 if (sk->sk_reuseport) { 422 tb->fastreuseport = FASTREUSEPORT_ANY; 423 tb->fastuid = uid; 424 tb->fast_rcv_saddr = sk->sk_rcv_saddr; 425 tb->fast_ipv6_only = ipv6_only_sock(sk); 426 tb->fast_sk_family = sk->sk_family; 427 #if IS_ENABLED(CONFIG_IPV6) 428 tb->fast_v6_rcv_saddr = sk->sk_v6_rcv_saddr; 429 #endif 430 } else { 431 tb->fastreuseport = 0; 432 } 433 } else { 434 if (!reuse) 435 tb->fastreuse = 0; 436 if (sk->sk_reuseport) { 437 /* We didn't match or we don't have fastreuseport set on 438 * the tb, but we have sk_reuseport set on this socket 439 * and we know that there are no bind conflicts with 440 * this socket in this tb, so reset our tb's reuseport 441 * settings so that any subsequent sockets that match 442 * our current socket will be put on the fast path. 443 * 444 * If we reset we need to set FASTREUSEPORT_STRICT so we 445 * do extra checking for all subsequent sk_reuseport 446 * socks. 447 */ 448 if (!sk_reuseport_match(tb, sk)) { 449 tb->fastreuseport = FASTREUSEPORT_STRICT; 450 tb->fastuid = uid; 451 tb->fast_rcv_saddr = sk->sk_rcv_saddr; 452 tb->fast_ipv6_only = ipv6_only_sock(sk); 453 tb->fast_sk_family = sk->sk_family; 454 #if IS_ENABLED(CONFIG_IPV6) 455 tb->fast_v6_rcv_saddr = sk->sk_v6_rcv_saddr; 456 #endif 457 } 458 } else { 459 tb->fastreuseport = 0; 460 } 461 } 462 } 463 464 /* Obtain a reference to a local port for the given sock, 465 * if snum is zero it means select any available local port. 466 * We try to allocate an odd port (and leave even ports for connect()) 467 */ 468 int inet_csk_get_port(struct sock *sk, unsigned short snum) 469 { 470 struct inet_hashinfo *hinfo = tcp_or_dccp_get_hashinfo(sk); 471 bool reuse = sk->sk_reuse && sk->sk_state != TCP_LISTEN; 472 bool found_port = false, check_bind_conflict = true; 473 bool bhash_created = false, bhash2_created = false; 474 struct inet_bind_hashbucket *head, *head2; 475 struct inet_bind2_bucket *tb2 = NULL; 476 struct inet_bind_bucket *tb = NULL; 477 bool head2_lock_acquired = false; 478 int ret = 1, port = snum, l3mdev; 479 struct net *net = sock_net(sk); 480 481 l3mdev = inet_sk_bound_l3mdev(sk); 482 483 if (!port) { 484 head = inet_csk_find_open_port(sk, &tb, &tb2, &head2, &port); 485 if (!head) 486 return ret; 487 488 head2_lock_acquired = true; 489 490 if (tb && tb2) 491 goto success; 492 found_port = true; 493 } else { 494 head = &hinfo->bhash[inet_bhashfn(net, port, 495 hinfo->bhash_size)]; 496 spin_lock_bh(&head->lock); 497 inet_bind_bucket_for_each(tb, &head->chain) 498 if (inet_bind_bucket_match(tb, net, port, l3mdev)) 499 break; 500 } 501 502 if (!tb) { 503 tb = inet_bind_bucket_create(hinfo->bind_bucket_cachep, net, 504 head, port, l3mdev); 505 if (!tb) 506 goto fail_unlock; 507 bhash_created = true; 508 } 509 510 if (!found_port) { 511 if (!hlist_empty(&tb->owners)) { 512 if (sk->sk_reuse == SK_FORCE_REUSE || 513 (tb->fastreuse > 0 && reuse) || 514 sk_reuseport_match(tb, sk)) 515 check_bind_conflict = false; 516 } 517 518 if (check_bind_conflict && inet_use_bhash2_on_bind(sk)) { 519 if (inet_bhash2_addr_any_conflict(sk, port, l3mdev, true, true)) 520 goto fail_unlock; 521 } 522 523 head2 = inet_bhashfn_portaddr(hinfo, sk, net, port); 524 spin_lock(&head2->lock); 525 head2_lock_acquired = true; 526 tb2 = inet_bind2_bucket_find(head2, net, port, l3mdev, sk); 527 } 528 529 if (!tb2) { 530 tb2 = inet_bind2_bucket_create(hinfo->bind2_bucket_cachep, 531 net, head2, port, l3mdev, sk); 532 if (!tb2) 533 goto fail_unlock; 534 bhash2_created = true; 535 } 536 537 if (!found_port && check_bind_conflict) { 538 if (inet_csk_bind_conflict(sk, tb, tb2, true, true)) 539 goto fail_unlock; 540 } 541 542 success: 543 inet_csk_update_fastreuse(tb, sk); 544 545 if (!inet_csk(sk)->icsk_bind_hash) 546 inet_bind_hash(sk, tb, tb2, port); 547 WARN_ON(inet_csk(sk)->icsk_bind_hash != tb); 548 WARN_ON(inet_csk(sk)->icsk_bind2_hash != tb2); 549 ret = 0; 550 551 fail_unlock: 552 if (ret) { 553 if (bhash_created) 554 inet_bind_bucket_destroy(hinfo->bind_bucket_cachep, tb); 555 if (bhash2_created) 556 inet_bind2_bucket_destroy(hinfo->bind2_bucket_cachep, 557 tb2); 558 } 559 if (head2_lock_acquired) 560 spin_unlock(&head2->lock); 561 spin_unlock_bh(&head->lock); 562 return ret; 563 } 564 EXPORT_SYMBOL_GPL(inet_csk_get_port); 565 566 /* 567 * Wait for an incoming connection, avoid race conditions. This must be called 568 * with the socket locked. 569 */ 570 static int inet_csk_wait_for_connect(struct sock *sk, long timeo) 571 { 572 struct inet_connection_sock *icsk = inet_csk(sk); 573 DEFINE_WAIT(wait); 574 int err; 575 576 /* 577 * True wake-one mechanism for incoming connections: only 578 * one process gets woken up, not the 'whole herd'. 579 * Since we do not 'race & poll' for established sockets 580 * anymore, the common case will execute the loop only once. 581 * 582 * Subtle issue: "add_wait_queue_exclusive()" will be added 583 * after any current non-exclusive waiters, and we know that 584 * it will always _stay_ after any new non-exclusive waiters 585 * because all non-exclusive waiters are added at the 586 * beginning of the wait-queue. As such, it's ok to "drop" 587 * our exclusiveness temporarily when we get woken up without 588 * having to remove and re-insert us on the wait queue. 589 */ 590 for (;;) { 591 prepare_to_wait_exclusive(sk_sleep(sk), &wait, 592 TASK_INTERRUPTIBLE); 593 release_sock(sk); 594 if (reqsk_queue_empty(&icsk->icsk_accept_queue)) 595 timeo = schedule_timeout(timeo); 596 sched_annotate_sleep(); 597 lock_sock(sk); 598 err = 0; 599 if (!reqsk_queue_empty(&icsk->icsk_accept_queue)) 600 break; 601 err = -EINVAL; 602 if (sk->sk_state != TCP_LISTEN) 603 break; 604 err = sock_intr_errno(timeo); 605 if (signal_pending(current)) 606 break; 607 err = -EAGAIN; 608 if (!timeo) 609 break; 610 } 611 finish_wait(sk_sleep(sk), &wait); 612 return err; 613 } 614 615 /* 616 * This will accept the next outstanding connection. 617 */ 618 struct sock *inet_csk_accept(struct sock *sk, int flags, int *err, bool kern) 619 { 620 struct inet_connection_sock *icsk = inet_csk(sk); 621 struct request_sock_queue *queue = &icsk->icsk_accept_queue; 622 struct request_sock *req; 623 struct sock *newsk; 624 int error; 625 626 lock_sock(sk); 627 628 /* We need to make sure that this socket is listening, 629 * and that it has something pending. 630 */ 631 error = -EINVAL; 632 if (sk->sk_state != TCP_LISTEN) 633 goto out_err; 634 635 /* Find already established connection */ 636 if (reqsk_queue_empty(queue)) { 637 long timeo = sock_rcvtimeo(sk, flags & O_NONBLOCK); 638 639 /* If this is a non blocking socket don't sleep */ 640 error = -EAGAIN; 641 if (!timeo) 642 goto out_err; 643 644 error = inet_csk_wait_for_connect(sk, timeo); 645 if (error) 646 goto out_err; 647 } 648 req = reqsk_queue_remove(queue, sk); 649 newsk = req->sk; 650 651 if (sk->sk_protocol == IPPROTO_TCP && 652 tcp_rsk(req)->tfo_listener) { 653 spin_lock_bh(&queue->fastopenq.lock); 654 if (tcp_rsk(req)->tfo_listener) { 655 /* We are still waiting for the final ACK from 3WHS 656 * so can't free req now. Instead, we set req->sk to 657 * NULL to signify that the child socket is taken 658 * so reqsk_fastopen_remove() will free the req 659 * when 3WHS finishes (or is aborted). 660 */ 661 req->sk = NULL; 662 req = NULL; 663 } 664 spin_unlock_bh(&queue->fastopenq.lock); 665 } 666 667 out: 668 release_sock(sk); 669 if (newsk && mem_cgroup_sockets_enabled) { 670 int amt; 671 672 /* atomically get the memory usage, set and charge the 673 * newsk->sk_memcg. 674 */ 675 lock_sock(newsk); 676 677 /* The socket has not been accepted yet, no need to look at 678 * newsk->sk_wmem_queued. 679 */ 680 amt = sk_mem_pages(newsk->sk_forward_alloc + 681 atomic_read(&newsk->sk_rmem_alloc)); 682 mem_cgroup_sk_alloc(newsk); 683 if (newsk->sk_memcg && amt) 684 mem_cgroup_charge_skmem(newsk->sk_memcg, amt, 685 GFP_KERNEL | __GFP_NOFAIL); 686 687 release_sock(newsk); 688 } 689 if (req) 690 reqsk_put(req); 691 return newsk; 692 out_err: 693 newsk = NULL; 694 req = NULL; 695 *err = error; 696 goto out; 697 } 698 EXPORT_SYMBOL(inet_csk_accept); 699 700 /* 701 * Using different timers for retransmit, delayed acks and probes 702 * We may wish use just one timer maintaining a list of expire jiffies 703 * to optimize. 704 */ 705 void inet_csk_init_xmit_timers(struct sock *sk, 706 void (*retransmit_handler)(struct timer_list *t), 707 void (*delack_handler)(struct timer_list *t), 708 void (*keepalive_handler)(struct timer_list *t)) 709 { 710 struct inet_connection_sock *icsk = inet_csk(sk); 711 712 timer_setup(&icsk->icsk_retransmit_timer, retransmit_handler, 0); 713 timer_setup(&icsk->icsk_delack_timer, delack_handler, 0); 714 timer_setup(&sk->sk_timer, keepalive_handler, 0); 715 icsk->icsk_pending = icsk->icsk_ack.pending = 0; 716 } 717 EXPORT_SYMBOL(inet_csk_init_xmit_timers); 718 719 void inet_csk_clear_xmit_timers(struct sock *sk) 720 { 721 struct inet_connection_sock *icsk = inet_csk(sk); 722 723 icsk->icsk_pending = icsk->icsk_ack.pending = 0; 724 725 sk_stop_timer(sk, &icsk->icsk_retransmit_timer); 726 sk_stop_timer(sk, &icsk->icsk_delack_timer); 727 sk_stop_timer(sk, &sk->sk_timer); 728 } 729 EXPORT_SYMBOL(inet_csk_clear_xmit_timers); 730 731 void inet_csk_delete_keepalive_timer(struct sock *sk) 732 { 733 sk_stop_timer(sk, &sk->sk_timer); 734 } 735 EXPORT_SYMBOL(inet_csk_delete_keepalive_timer); 736 737 void inet_csk_reset_keepalive_timer(struct sock *sk, unsigned long len) 738 { 739 sk_reset_timer(sk, &sk->sk_timer, jiffies + len); 740 } 741 EXPORT_SYMBOL(inet_csk_reset_keepalive_timer); 742 743 struct dst_entry *inet_csk_route_req(const struct sock *sk, 744 struct flowi4 *fl4, 745 const struct request_sock *req) 746 { 747 const struct inet_request_sock *ireq = inet_rsk(req); 748 struct net *net = read_pnet(&ireq->ireq_net); 749 struct ip_options_rcu *opt; 750 struct rtable *rt; 751 752 rcu_read_lock(); 753 opt = rcu_dereference(ireq->ireq_opt); 754 755 flowi4_init_output(fl4, ireq->ir_iif, ireq->ir_mark, 756 RT_CONN_FLAGS(sk), RT_SCOPE_UNIVERSE, 757 sk->sk_protocol, inet_sk_flowi_flags(sk), 758 (opt && opt->opt.srr) ? opt->opt.faddr : ireq->ir_rmt_addr, 759 ireq->ir_loc_addr, ireq->ir_rmt_port, 760 htons(ireq->ir_num), sk->sk_uid); 761 security_req_classify_flow(req, flowi4_to_flowi_common(fl4)); 762 rt = ip_route_output_flow(net, fl4, sk); 763 if (IS_ERR(rt)) 764 goto no_route; 765 if (opt && opt->opt.is_strictroute && rt->rt_uses_gateway) 766 goto route_err; 767 rcu_read_unlock(); 768 return &rt->dst; 769 770 route_err: 771 ip_rt_put(rt); 772 no_route: 773 rcu_read_unlock(); 774 __IP_INC_STATS(net, IPSTATS_MIB_OUTNOROUTES); 775 return NULL; 776 } 777 EXPORT_SYMBOL_GPL(inet_csk_route_req); 778 779 struct dst_entry *inet_csk_route_child_sock(const struct sock *sk, 780 struct sock *newsk, 781 const struct request_sock *req) 782 { 783 const struct inet_request_sock *ireq = inet_rsk(req); 784 struct net *net = read_pnet(&ireq->ireq_net); 785 struct inet_sock *newinet = inet_sk(newsk); 786 struct ip_options_rcu *opt; 787 struct flowi4 *fl4; 788 struct rtable *rt; 789 790 opt = rcu_dereference(ireq->ireq_opt); 791 fl4 = &newinet->cork.fl.u.ip4; 792 793 flowi4_init_output(fl4, ireq->ir_iif, ireq->ir_mark, 794 RT_CONN_FLAGS(sk), RT_SCOPE_UNIVERSE, 795 sk->sk_protocol, inet_sk_flowi_flags(sk), 796 (opt && opt->opt.srr) ? opt->opt.faddr : ireq->ir_rmt_addr, 797 ireq->ir_loc_addr, ireq->ir_rmt_port, 798 htons(ireq->ir_num), sk->sk_uid); 799 security_req_classify_flow(req, flowi4_to_flowi_common(fl4)); 800 rt = ip_route_output_flow(net, fl4, sk); 801 if (IS_ERR(rt)) 802 goto no_route; 803 if (opt && opt->opt.is_strictroute && rt->rt_uses_gateway) 804 goto route_err; 805 return &rt->dst; 806 807 route_err: 808 ip_rt_put(rt); 809 no_route: 810 __IP_INC_STATS(net, IPSTATS_MIB_OUTNOROUTES); 811 return NULL; 812 } 813 EXPORT_SYMBOL_GPL(inet_csk_route_child_sock); 814 815 /* Decide when to expire the request and when to resend SYN-ACK */ 816 static void syn_ack_recalc(struct request_sock *req, 817 const int max_syn_ack_retries, 818 const u8 rskq_defer_accept, 819 int *expire, int *resend) 820 { 821 if (!rskq_defer_accept) { 822 *expire = req->num_timeout >= max_syn_ack_retries; 823 *resend = 1; 824 return; 825 } 826 *expire = req->num_timeout >= max_syn_ack_retries && 827 (!inet_rsk(req)->acked || req->num_timeout >= rskq_defer_accept); 828 /* Do not resend while waiting for data after ACK, 829 * start to resend on end of deferring period to give 830 * last chance for data or ACK to create established socket. 831 */ 832 *resend = !inet_rsk(req)->acked || 833 req->num_timeout >= rskq_defer_accept - 1; 834 } 835 836 int inet_rtx_syn_ack(const struct sock *parent, struct request_sock *req) 837 { 838 int err = req->rsk_ops->rtx_syn_ack(parent, req); 839 840 if (!err) 841 req->num_retrans++; 842 return err; 843 } 844 EXPORT_SYMBOL(inet_rtx_syn_ack); 845 846 static struct request_sock *inet_reqsk_clone(struct request_sock *req, 847 struct sock *sk) 848 { 849 struct sock *req_sk, *nreq_sk; 850 struct request_sock *nreq; 851 852 nreq = kmem_cache_alloc(req->rsk_ops->slab, GFP_ATOMIC | __GFP_NOWARN); 853 if (!nreq) { 854 __NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPMIGRATEREQFAILURE); 855 856 /* paired with refcount_inc_not_zero() in reuseport_migrate_sock() */ 857 sock_put(sk); 858 return NULL; 859 } 860 861 req_sk = req_to_sk(req); 862 nreq_sk = req_to_sk(nreq); 863 864 memcpy(nreq_sk, req_sk, 865 offsetof(struct sock, sk_dontcopy_begin)); 866 memcpy(&nreq_sk->sk_dontcopy_end, &req_sk->sk_dontcopy_end, 867 req->rsk_ops->obj_size - offsetof(struct sock, sk_dontcopy_end)); 868 869 sk_node_init(&nreq_sk->sk_node); 870 nreq_sk->sk_tx_queue_mapping = req_sk->sk_tx_queue_mapping; 871 #ifdef CONFIG_SOCK_RX_QUEUE_MAPPING 872 nreq_sk->sk_rx_queue_mapping = req_sk->sk_rx_queue_mapping; 873 #endif 874 nreq_sk->sk_incoming_cpu = req_sk->sk_incoming_cpu; 875 876 nreq->rsk_listener = sk; 877 878 /* We need not acquire fastopenq->lock 879 * because the child socket is locked in inet_csk_listen_stop(). 880 */ 881 if (sk->sk_protocol == IPPROTO_TCP && tcp_rsk(nreq)->tfo_listener) 882 rcu_assign_pointer(tcp_sk(nreq->sk)->fastopen_rsk, nreq); 883 884 return nreq; 885 } 886 887 static void reqsk_queue_migrated(struct request_sock_queue *queue, 888 const struct request_sock *req) 889 { 890 if (req->num_timeout == 0) 891 atomic_inc(&queue->young); 892 atomic_inc(&queue->qlen); 893 } 894 895 static void reqsk_migrate_reset(struct request_sock *req) 896 { 897 req->saved_syn = NULL; 898 #if IS_ENABLED(CONFIG_IPV6) 899 inet_rsk(req)->ipv6_opt = NULL; 900 inet_rsk(req)->pktopts = NULL; 901 #else 902 inet_rsk(req)->ireq_opt = NULL; 903 #endif 904 } 905 906 /* return true if req was found in the ehash table */ 907 static bool reqsk_queue_unlink(struct request_sock *req) 908 { 909 struct sock *sk = req_to_sk(req); 910 bool found = false; 911 912 if (sk_hashed(sk)) { 913 struct inet_hashinfo *hashinfo = tcp_or_dccp_get_hashinfo(sk); 914 spinlock_t *lock = inet_ehash_lockp(hashinfo, req->rsk_hash); 915 916 spin_lock(lock); 917 found = __sk_nulls_del_node_init_rcu(sk); 918 spin_unlock(lock); 919 } 920 if (timer_pending(&req->rsk_timer) && del_timer_sync(&req->rsk_timer)) 921 reqsk_put(req); 922 return found; 923 } 924 925 bool inet_csk_reqsk_queue_drop(struct sock *sk, struct request_sock *req) 926 { 927 bool unlinked = reqsk_queue_unlink(req); 928 929 if (unlinked) { 930 reqsk_queue_removed(&inet_csk(sk)->icsk_accept_queue, req); 931 reqsk_put(req); 932 } 933 return unlinked; 934 } 935 EXPORT_SYMBOL(inet_csk_reqsk_queue_drop); 936 937 void inet_csk_reqsk_queue_drop_and_put(struct sock *sk, struct request_sock *req) 938 { 939 inet_csk_reqsk_queue_drop(sk, req); 940 reqsk_put(req); 941 } 942 EXPORT_SYMBOL(inet_csk_reqsk_queue_drop_and_put); 943 944 static void reqsk_timer_handler(struct timer_list *t) 945 { 946 struct request_sock *req = from_timer(req, t, rsk_timer); 947 struct request_sock *nreq = NULL, *oreq = req; 948 struct sock *sk_listener = req->rsk_listener; 949 struct inet_connection_sock *icsk; 950 struct request_sock_queue *queue; 951 struct net *net; 952 int max_syn_ack_retries, qlen, expire = 0, resend = 0; 953 954 if (inet_sk_state_load(sk_listener) != TCP_LISTEN) { 955 struct sock *nsk; 956 957 nsk = reuseport_migrate_sock(sk_listener, req_to_sk(req), NULL); 958 if (!nsk) 959 goto drop; 960 961 nreq = inet_reqsk_clone(req, nsk); 962 if (!nreq) 963 goto drop; 964 965 /* The new timer for the cloned req can decrease the 2 966 * by calling inet_csk_reqsk_queue_drop_and_put(), so 967 * hold another count to prevent use-after-free and 968 * call reqsk_put() just before return. 969 */ 970 refcount_set(&nreq->rsk_refcnt, 2 + 1); 971 timer_setup(&nreq->rsk_timer, reqsk_timer_handler, TIMER_PINNED); 972 reqsk_queue_migrated(&inet_csk(nsk)->icsk_accept_queue, req); 973 974 req = nreq; 975 sk_listener = nsk; 976 } 977 978 icsk = inet_csk(sk_listener); 979 net = sock_net(sk_listener); 980 max_syn_ack_retries = icsk->icsk_syn_retries ? : 981 READ_ONCE(net->ipv4.sysctl_tcp_synack_retries); 982 /* Normally all the openreqs are young and become mature 983 * (i.e. converted to established socket) for first timeout. 984 * If synack was not acknowledged for 1 second, it means 985 * one of the following things: synack was lost, ack was lost, 986 * rtt is high or nobody planned to ack (i.e. synflood). 987 * When server is a bit loaded, queue is populated with old 988 * open requests, reducing effective size of queue. 989 * When server is well loaded, queue size reduces to zero 990 * after several minutes of work. It is not synflood, 991 * it is normal operation. The solution is pruning 992 * too old entries overriding normal timeout, when 993 * situation becomes dangerous. 994 * 995 * Essentially, we reserve half of room for young 996 * embrions; and abort old ones without pity, if old 997 * ones are about to clog our table. 998 */ 999 queue = &icsk->icsk_accept_queue; 1000 qlen = reqsk_queue_len(queue); 1001 if ((qlen << 1) > max(8U, READ_ONCE(sk_listener->sk_max_ack_backlog))) { 1002 int young = reqsk_queue_len_young(queue) << 1; 1003 1004 while (max_syn_ack_retries > 2) { 1005 if (qlen < young) 1006 break; 1007 max_syn_ack_retries--; 1008 young <<= 1; 1009 } 1010 } 1011 syn_ack_recalc(req, max_syn_ack_retries, READ_ONCE(queue->rskq_defer_accept), 1012 &expire, &resend); 1013 req->rsk_ops->syn_ack_timeout(req); 1014 if (!expire && 1015 (!resend || 1016 !inet_rtx_syn_ack(sk_listener, req) || 1017 inet_rsk(req)->acked)) { 1018 if (req->num_timeout++ == 0) 1019 atomic_dec(&queue->young); 1020 mod_timer(&req->rsk_timer, jiffies + reqsk_timeout(req, TCP_RTO_MAX)); 1021 1022 if (!nreq) 1023 return; 1024 1025 if (!inet_ehash_insert(req_to_sk(nreq), req_to_sk(oreq), NULL)) { 1026 /* delete timer */ 1027 inet_csk_reqsk_queue_drop(sk_listener, nreq); 1028 goto no_ownership; 1029 } 1030 1031 __NET_INC_STATS(net, LINUX_MIB_TCPMIGRATEREQSUCCESS); 1032 reqsk_migrate_reset(oreq); 1033 reqsk_queue_removed(&inet_csk(oreq->rsk_listener)->icsk_accept_queue, oreq); 1034 reqsk_put(oreq); 1035 1036 reqsk_put(nreq); 1037 return; 1038 } 1039 1040 /* Even if we can clone the req, we may need not retransmit any more 1041 * SYN+ACKs (nreq->num_timeout > max_syn_ack_retries, etc), or another 1042 * CPU may win the "own_req" race so that inet_ehash_insert() fails. 1043 */ 1044 if (nreq) { 1045 __NET_INC_STATS(net, LINUX_MIB_TCPMIGRATEREQFAILURE); 1046 no_ownership: 1047 reqsk_migrate_reset(nreq); 1048 reqsk_queue_removed(queue, nreq); 1049 __reqsk_free(nreq); 1050 } 1051 1052 drop: 1053 inet_csk_reqsk_queue_drop_and_put(oreq->rsk_listener, oreq); 1054 } 1055 1056 static void reqsk_queue_hash_req(struct request_sock *req, 1057 unsigned long timeout) 1058 { 1059 timer_setup(&req->rsk_timer, reqsk_timer_handler, TIMER_PINNED); 1060 mod_timer(&req->rsk_timer, jiffies + timeout); 1061 1062 inet_ehash_insert(req_to_sk(req), NULL, NULL); 1063 /* before letting lookups find us, make sure all req fields 1064 * are committed to memory and refcnt initialized. 1065 */ 1066 smp_wmb(); 1067 refcount_set(&req->rsk_refcnt, 2 + 1); 1068 } 1069 1070 void inet_csk_reqsk_queue_hash_add(struct sock *sk, struct request_sock *req, 1071 unsigned long timeout) 1072 { 1073 reqsk_queue_hash_req(req, timeout); 1074 inet_csk_reqsk_queue_added(sk); 1075 } 1076 EXPORT_SYMBOL_GPL(inet_csk_reqsk_queue_hash_add); 1077 1078 static void inet_clone_ulp(const struct request_sock *req, struct sock *newsk, 1079 const gfp_t priority) 1080 { 1081 struct inet_connection_sock *icsk = inet_csk(newsk); 1082 1083 if (!icsk->icsk_ulp_ops) 1084 return; 1085 1086 if (icsk->icsk_ulp_ops->clone) 1087 icsk->icsk_ulp_ops->clone(req, newsk, priority); 1088 } 1089 1090 /** 1091 * inet_csk_clone_lock - clone an inet socket, and lock its clone 1092 * @sk: the socket to clone 1093 * @req: request_sock 1094 * @priority: for allocation (%GFP_KERNEL, %GFP_ATOMIC, etc) 1095 * 1096 * Caller must unlock socket even in error path (bh_unlock_sock(newsk)) 1097 */ 1098 struct sock *inet_csk_clone_lock(const struct sock *sk, 1099 const struct request_sock *req, 1100 const gfp_t priority) 1101 { 1102 struct sock *newsk = sk_clone_lock(sk, priority); 1103 1104 if (newsk) { 1105 struct inet_connection_sock *newicsk = inet_csk(newsk); 1106 1107 inet_sk_set_state(newsk, TCP_SYN_RECV); 1108 newicsk->icsk_bind_hash = NULL; 1109 newicsk->icsk_bind2_hash = NULL; 1110 1111 inet_sk(newsk)->inet_dport = inet_rsk(req)->ir_rmt_port; 1112 inet_sk(newsk)->inet_num = inet_rsk(req)->ir_num; 1113 inet_sk(newsk)->inet_sport = htons(inet_rsk(req)->ir_num); 1114 1115 /* listeners have SOCK_RCU_FREE, not the children */ 1116 sock_reset_flag(newsk, SOCK_RCU_FREE); 1117 1118 inet_sk(newsk)->mc_list = NULL; 1119 1120 newsk->sk_mark = inet_rsk(req)->ir_mark; 1121 atomic64_set(&newsk->sk_cookie, 1122 atomic64_read(&inet_rsk(req)->ir_cookie)); 1123 1124 newicsk->icsk_retransmits = 0; 1125 newicsk->icsk_backoff = 0; 1126 newicsk->icsk_probes_out = 0; 1127 newicsk->icsk_probes_tstamp = 0; 1128 1129 /* Deinitialize accept_queue to trap illegal accesses. */ 1130 memset(&newicsk->icsk_accept_queue, 0, sizeof(newicsk->icsk_accept_queue)); 1131 1132 inet_clone_ulp(req, newsk, priority); 1133 1134 security_inet_csk_clone(newsk, req); 1135 } 1136 return newsk; 1137 } 1138 EXPORT_SYMBOL_GPL(inet_csk_clone_lock); 1139 1140 /* 1141 * At this point, there should be no process reference to this 1142 * socket, and thus no user references at all. Therefore we 1143 * can assume the socket waitqueue is inactive and nobody will 1144 * try to jump onto it. 1145 */ 1146 void inet_csk_destroy_sock(struct sock *sk) 1147 { 1148 WARN_ON(sk->sk_state != TCP_CLOSE); 1149 WARN_ON(!sock_flag(sk, SOCK_DEAD)); 1150 1151 /* It cannot be in hash table! */ 1152 WARN_ON(!sk_unhashed(sk)); 1153 1154 /* If it has not 0 inet_sk(sk)->inet_num, it must be bound */ 1155 WARN_ON(inet_sk(sk)->inet_num && !inet_csk(sk)->icsk_bind_hash); 1156 1157 sk->sk_prot->destroy(sk); 1158 1159 sk_stream_kill_queues(sk); 1160 1161 xfrm_sk_free_policy(sk); 1162 1163 sk_refcnt_debug_release(sk); 1164 1165 this_cpu_dec(*sk->sk_prot->orphan_count); 1166 1167 sock_put(sk); 1168 } 1169 EXPORT_SYMBOL(inet_csk_destroy_sock); 1170 1171 /* This function allows to force a closure of a socket after the call to 1172 * tcp/dccp_create_openreq_child(). 1173 */ 1174 void inet_csk_prepare_forced_close(struct sock *sk) 1175 __releases(&sk->sk_lock.slock) 1176 { 1177 /* sk_clone_lock locked the socket and set refcnt to 2 */ 1178 bh_unlock_sock(sk); 1179 sock_put(sk); 1180 inet_csk_prepare_for_destroy_sock(sk); 1181 inet_sk(sk)->inet_num = 0; 1182 } 1183 EXPORT_SYMBOL(inet_csk_prepare_forced_close); 1184 1185 int inet_csk_listen_start(struct sock *sk) 1186 { 1187 struct inet_connection_sock *icsk = inet_csk(sk); 1188 struct inet_sock *inet = inet_sk(sk); 1189 int err = -EADDRINUSE; 1190 1191 reqsk_queue_alloc(&icsk->icsk_accept_queue); 1192 1193 sk->sk_ack_backlog = 0; 1194 inet_csk_delack_init(sk); 1195 1196 if (sk->sk_txrehash == SOCK_TXREHASH_DEFAULT) 1197 sk->sk_txrehash = READ_ONCE(sock_net(sk)->core.sysctl_txrehash); 1198 1199 /* There is race window here: we announce ourselves listening, 1200 * but this transition is still not validated by get_port(). 1201 * It is OK, because this socket enters to hash table only 1202 * after validation is complete. 1203 */ 1204 inet_sk_state_store(sk, TCP_LISTEN); 1205 if (!sk->sk_prot->get_port(sk, inet->inet_num)) { 1206 inet->inet_sport = htons(inet->inet_num); 1207 1208 sk_dst_reset(sk); 1209 err = sk->sk_prot->hash(sk); 1210 1211 if (likely(!err)) 1212 return 0; 1213 } 1214 1215 inet_sk_set_state(sk, TCP_CLOSE); 1216 return err; 1217 } 1218 EXPORT_SYMBOL_GPL(inet_csk_listen_start); 1219 1220 static void inet_child_forget(struct sock *sk, struct request_sock *req, 1221 struct sock *child) 1222 { 1223 sk->sk_prot->disconnect(child, O_NONBLOCK); 1224 1225 sock_orphan(child); 1226 1227 this_cpu_inc(*sk->sk_prot->orphan_count); 1228 1229 if (sk->sk_protocol == IPPROTO_TCP && tcp_rsk(req)->tfo_listener) { 1230 BUG_ON(rcu_access_pointer(tcp_sk(child)->fastopen_rsk) != req); 1231 BUG_ON(sk != req->rsk_listener); 1232 1233 /* Paranoid, to prevent race condition if 1234 * an inbound pkt destined for child is 1235 * blocked by sock lock in tcp_v4_rcv(). 1236 * Also to satisfy an assertion in 1237 * tcp_v4_destroy_sock(). 1238 */ 1239 RCU_INIT_POINTER(tcp_sk(child)->fastopen_rsk, NULL); 1240 } 1241 inet_csk_destroy_sock(child); 1242 } 1243 1244 struct sock *inet_csk_reqsk_queue_add(struct sock *sk, 1245 struct request_sock *req, 1246 struct sock *child) 1247 { 1248 struct request_sock_queue *queue = &inet_csk(sk)->icsk_accept_queue; 1249 1250 spin_lock(&queue->rskq_lock); 1251 if (unlikely(sk->sk_state != TCP_LISTEN)) { 1252 inet_child_forget(sk, req, child); 1253 child = NULL; 1254 } else { 1255 req->sk = child; 1256 req->dl_next = NULL; 1257 if (queue->rskq_accept_head == NULL) 1258 WRITE_ONCE(queue->rskq_accept_head, req); 1259 else 1260 queue->rskq_accept_tail->dl_next = req; 1261 queue->rskq_accept_tail = req; 1262 sk_acceptq_added(sk); 1263 } 1264 spin_unlock(&queue->rskq_lock); 1265 return child; 1266 } 1267 EXPORT_SYMBOL(inet_csk_reqsk_queue_add); 1268 1269 struct sock *inet_csk_complete_hashdance(struct sock *sk, struct sock *child, 1270 struct request_sock *req, bool own_req) 1271 { 1272 if (own_req) { 1273 inet_csk_reqsk_queue_drop(req->rsk_listener, req); 1274 reqsk_queue_removed(&inet_csk(req->rsk_listener)->icsk_accept_queue, req); 1275 1276 if (sk != req->rsk_listener) { 1277 /* another listening sk has been selected, 1278 * migrate the req to it. 1279 */ 1280 struct request_sock *nreq; 1281 1282 /* hold a refcnt for the nreq->rsk_listener 1283 * which is assigned in inet_reqsk_clone() 1284 */ 1285 sock_hold(sk); 1286 nreq = inet_reqsk_clone(req, sk); 1287 if (!nreq) { 1288 inet_child_forget(sk, req, child); 1289 goto child_put; 1290 } 1291 1292 refcount_set(&nreq->rsk_refcnt, 1); 1293 if (inet_csk_reqsk_queue_add(sk, nreq, child)) { 1294 __NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPMIGRATEREQSUCCESS); 1295 reqsk_migrate_reset(req); 1296 reqsk_put(req); 1297 return child; 1298 } 1299 1300 __NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPMIGRATEREQFAILURE); 1301 reqsk_migrate_reset(nreq); 1302 __reqsk_free(nreq); 1303 } else if (inet_csk_reqsk_queue_add(sk, req, child)) { 1304 return child; 1305 } 1306 } 1307 /* Too bad, another child took ownership of the request, undo. */ 1308 child_put: 1309 bh_unlock_sock(child); 1310 sock_put(child); 1311 return NULL; 1312 } 1313 EXPORT_SYMBOL(inet_csk_complete_hashdance); 1314 1315 /* 1316 * This routine closes sockets which have been at least partially 1317 * opened, but not yet accepted. 1318 */ 1319 void inet_csk_listen_stop(struct sock *sk) 1320 { 1321 struct inet_connection_sock *icsk = inet_csk(sk); 1322 struct request_sock_queue *queue = &icsk->icsk_accept_queue; 1323 struct request_sock *next, *req; 1324 1325 /* Following specs, it would be better either to send FIN 1326 * (and enter FIN-WAIT-1, it is normal close) 1327 * or to send active reset (abort). 1328 * Certainly, it is pretty dangerous while synflood, but it is 1329 * bad justification for our negligence 8) 1330 * To be honest, we are not able to make either 1331 * of the variants now. --ANK 1332 */ 1333 while ((req = reqsk_queue_remove(queue, sk)) != NULL) { 1334 struct sock *child = req->sk, *nsk; 1335 struct request_sock *nreq; 1336 1337 local_bh_disable(); 1338 bh_lock_sock(child); 1339 WARN_ON(sock_owned_by_user(child)); 1340 sock_hold(child); 1341 1342 nsk = reuseport_migrate_sock(sk, child, NULL); 1343 if (nsk) { 1344 nreq = inet_reqsk_clone(req, nsk); 1345 if (nreq) { 1346 refcount_set(&nreq->rsk_refcnt, 1); 1347 1348 if (inet_csk_reqsk_queue_add(nsk, nreq, child)) { 1349 __NET_INC_STATS(sock_net(nsk), 1350 LINUX_MIB_TCPMIGRATEREQSUCCESS); 1351 reqsk_migrate_reset(req); 1352 } else { 1353 __NET_INC_STATS(sock_net(nsk), 1354 LINUX_MIB_TCPMIGRATEREQFAILURE); 1355 reqsk_migrate_reset(nreq); 1356 __reqsk_free(nreq); 1357 } 1358 1359 /* inet_csk_reqsk_queue_add() has already 1360 * called inet_child_forget() on failure case. 1361 */ 1362 goto skip_child_forget; 1363 } 1364 } 1365 1366 inet_child_forget(sk, req, child); 1367 skip_child_forget: 1368 reqsk_put(req); 1369 bh_unlock_sock(child); 1370 local_bh_enable(); 1371 sock_put(child); 1372 1373 cond_resched(); 1374 } 1375 if (queue->fastopenq.rskq_rst_head) { 1376 /* Free all the reqs queued in rskq_rst_head. */ 1377 spin_lock_bh(&queue->fastopenq.lock); 1378 req = queue->fastopenq.rskq_rst_head; 1379 queue->fastopenq.rskq_rst_head = NULL; 1380 spin_unlock_bh(&queue->fastopenq.lock); 1381 while (req != NULL) { 1382 next = req->dl_next; 1383 reqsk_put(req); 1384 req = next; 1385 } 1386 } 1387 WARN_ON_ONCE(sk->sk_ack_backlog); 1388 } 1389 EXPORT_SYMBOL_GPL(inet_csk_listen_stop); 1390 1391 void inet_csk_addr2sockaddr(struct sock *sk, struct sockaddr *uaddr) 1392 { 1393 struct sockaddr_in *sin = (struct sockaddr_in *)uaddr; 1394 const struct inet_sock *inet = inet_sk(sk); 1395 1396 sin->sin_family = AF_INET; 1397 sin->sin_addr.s_addr = inet->inet_daddr; 1398 sin->sin_port = inet->inet_dport; 1399 } 1400 EXPORT_SYMBOL_GPL(inet_csk_addr2sockaddr); 1401 1402 static struct dst_entry *inet_csk_rebuild_route(struct sock *sk, struct flowi *fl) 1403 { 1404 const struct inet_sock *inet = inet_sk(sk); 1405 const struct ip_options_rcu *inet_opt; 1406 __be32 daddr = inet->inet_daddr; 1407 struct flowi4 *fl4; 1408 struct rtable *rt; 1409 1410 rcu_read_lock(); 1411 inet_opt = rcu_dereference(inet->inet_opt); 1412 if (inet_opt && inet_opt->opt.srr) 1413 daddr = inet_opt->opt.faddr; 1414 fl4 = &fl->u.ip4; 1415 rt = ip_route_output_ports(sock_net(sk), fl4, sk, daddr, 1416 inet->inet_saddr, inet->inet_dport, 1417 inet->inet_sport, sk->sk_protocol, 1418 RT_CONN_FLAGS(sk), sk->sk_bound_dev_if); 1419 if (IS_ERR(rt)) 1420 rt = NULL; 1421 if (rt) 1422 sk_setup_caps(sk, &rt->dst); 1423 rcu_read_unlock(); 1424 1425 return &rt->dst; 1426 } 1427 1428 struct dst_entry *inet_csk_update_pmtu(struct sock *sk, u32 mtu) 1429 { 1430 struct dst_entry *dst = __sk_dst_check(sk, 0); 1431 struct inet_sock *inet = inet_sk(sk); 1432 1433 if (!dst) { 1434 dst = inet_csk_rebuild_route(sk, &inet->cork.fl); 1435 if (!dst) 1436 goto out; 1437 } 1438 dst->ops->update_pmtu(dst, sk, NULL, mtu, true); 1439 1440 dst = __sk_dst_check(sk, 0); 1441 if (!dst) 1442 dst = inet_csk_rebuild_route(sk, &inet->cork.fl); 1443 out: 1444 return dst; 1445 } 1446 EXPORT_SYMBOL_GPL(inet_csk_update_pmtu); 1447