1 /* 2 * INET An implementation of the TCP/IP protocol suite for the LINUX 3 * operating system. INET is implemented using the BSD Socket 4 * interface as the means of communication with the user level. 5 * 6 * Support for INET connection oriented protocols. 7 * 8 * Authors: See the TCP sources 9 * 10 * This program is free software; you can redistribute it and/or 11 * modify it under the terms of the GNU General Public License 12 * as published by the Free Software Foundation; either version 13 * 2 of the License, or(at your option) any later version. 14 */ 15 16 #include <linux/module.h> 17 #include <linux/jhash.h> 18 19 #include <net/inet_connection_sock.h> 20 #include <net/inet_hashtables.h> 21 #include <net/inet_timewait_sock.h> 22 #include <net/ip.h> 23 #include <net/route.h> 24 #include <net/tcp_states.h> 25 #include <net/xfrm.h> 26 #include <net/tcp.h> 27 #include <net/sock_reuseport.h> 28 29 #ifdef INET_CSK_DEBUG 30 const char inet_csk_timer_bug_msg[] = "inet_csk BUG: unknown timer value\n"; 31 EXPORT_SYMBOL(inet_csk_timer_bug_msg); 32 #endif 33 34 void inet_get_local_port_range(struct net *net, int *low, int *high) 35 { 36 unsigned int seq; 37 38 do { 39 seq = read_seqbegin(&net->ipv4.ip_local_ports.lock); 40 41 *low = net->ipv4.ip_local_ports.range[0]; 42 *high = net->ipv4.ip_local_ports.range[1]; 43 } while (read_seqretry(&net->ipv4.ip_local_ports.lock, seq)); 44 } 45 EXPORT_SYMBOL(inet_get_local_port_range); 46 47 int inet_csk_bind_conflict(const struct sock *sk, 48 const struct inet_bind_bucket *tb, bool relax) 49 { 50 struct sock *sk2; 51 int reuse = sk->sk_reuse; 52 int reuseport = sk->sk_reuseport; 53 kuid_t uid = sock_i_uid((struct sock *)sk); 54 55 /* 56 * Unlike other sk lookup places we do not check 57 * for sk_net here, since _all_ the socks listed 58 * in tb->owners list belong to the same net - the 59 * one this bucket belongs to. 60 */ 61 62 sk_for_each_bound(sk2, &tb->owners) { 63 if (sk != sk2 && 64 !inet_v6_ipv6only(sk2) && 65 (!sk->sk_bound_dev_if || 66 !sk2->sk_bound_dev_if || 67 sk->sk_bound_dev_if == sk2->sk_bound_dev_if)) { 68 if ((!reuse || !sk2->sk_reuse || 69 sk2->sk_state == TCP_LISTEN) && 70 (!reuseport || !sk2->sk_reuseport || 71 rcu_access_pointer(sk->sk_reuseport_cb) || 72 (sk2->sk_state != TCP_TIME_WAIT && 73 !uid_eq(uid, sock_i_uid(sk2))))) { 74 75 if (!sk2->sk_rcv_saddr || !sk->sk_rcv_saddr || 76 sk2->sk_rcv_saddr == sk->sk_rcv_saddr) 77 break; 78 } 79 if (!relax && reuse && sk2->sk_reuse && 80 sk2->sk_state != TCP_LISTEN) { 81 82 if (!sk2->sk_rcv_saddr || !sk->sk_rcv_saddr || 83 sk2->sk_rcv_saddr == sk->sk_rcv_saddr) 84 break; 85 } 86 } 87 } 88 return sk2 != NULL; 89 } 90 EXPORT_SYMBOL_GPL(inet_csk_bind_conflict); 91 92 /* Obtain a reference to a local port for the given sock, 93 * if snum is zero it means select any available local port. 94 * We try to allocate an odd port (and leave even ports for connect()) 95 */ 96 int inet_csk_get_port(struct sock *sk, unsigned short snum) 97 { 98 bool reuse = sk->sk_reuse && sk->sk_state != TCP_LISTEN; 99 struct inet_hashinfo *hinfo = sk->sk_prot->h.hashinfo; 100 int ret = 1, attempts = 5, port = snum; 101 int smallest_size = -1, smallest_port; 102 struct inet_bind_hashbucket *head; 103 struct net *net = sock_net(sk); 104 int i, low, high, attempt_half; 105 struct inet_bind_bucket *tb; 106 kuid_t uid = sock_i_uid(sk); 107 u32 remaining, offset; 108 109 if (port) { 110 have_port: 111 head = &hinfo->bhash[inet_bhashfn(net, port, 112 hinfo->bhash_size)]; 113 spin_lock_bh(&head->lock); 114 inet_bind_bucket_for_each(tb, &head->chain) 115 if (net_eq(ib_net(tb), net) && tb->port == port) 116 goto tb_found; 117 118 goto tb_not_found; 119 } 120 again: 121 attempt_half = (sk->sk_reuse == SK_CAN_REUSE) ? 1 : 0; 122 other_half_scan: 123 inet_get_local_port_range(net, &low, &high); 124 high++; /* [32768, 60999] -> [32768, 61000[ */ 125 if (high - low < 4) 126 attempt_half = 0; 127 if (attempt_half) { 128 int half = low + (((high - low) >> 2) << 1); 129 130 if (attempt_half == 1) 131 high = half; 132 else 133 low = half; 134 } 135 remaining = high - low; 136 if (likely(remaining > 1)) 137 remaining &= ~1U; 138 139 offset = prandom_u32() % remaining; 140 /* __inet_hash_connect() favors ports having @low parity 141 * We do the opposite to not pollute connect() users. 142 */ 143 offset |= 1U; 144 smallest_size = -1; 145 smallest_port = low; /* avoid compiler warning */ 146 147 other_parity_scan: 148 port = low + offset; 149 for (i = 0; i < remaining; i += 2, port += 2) { 150 if (unlikely(port >= high)) 151 port -= remaining; 152 if (inet_is_local_reserved_port(net, port)) 153 continue; 154 head = &hinfo->bhash[inet_bhashfn(net, port, 155 hinfo->bhash_size)]; 156 spin_lock_bh(&head->lock); 157 inet_bind_bucket_for_each(tb, &head->chain) 158 if (net_eq(ib_net(tb), net) && tb->port == port) { 159 if (((tb->fastreuse > 0 && reuse) || 160 (tb->fastreuseport > 0 && 161 sk->sk_reuseport && 162 !rcu_access_pointer(sk->sk_reuseport_cb) && 163 uid_eq(tb->fastuid, uid))) && 164 (tb->num_owners < smallest_size || smallest_size == -1)) { 165 smallest_size = tb->num_owners; 166 smallest_port = port; 167 } 168 if (!inet_csk(sk)->icsk_af_ops->bind_conflict(sk, tb, false)) 169 goto tb_found; 170 goto next_port; 171 } 172 goto tb_not_found; 173 next_port: 174 spin_unlock_bh(&head->lock); 175 cond_resched(); 176 } 177 178 if (smallest_size != -1) { 179 port = smallest_port; 180 goto have_port; 181 } 182 offset--; 183 if (!(offset & 1)) 184 goto other_parity_scan; 185 186 if (attempt_half == 1) { 187 /* OK we now try the upper half of the range */ 188 attempt_half = 2; 189 goto other_half_scan; 190 } 191 return ret; 192 193 tb_not_found: 194 tb = inet_bind_bucket_create(hinfo->bind_bucket_cachep, 195 net, head, port); 196 if (!tb) 197 goto fail_unlock; 198 tb_found: 199 if (!hlist_empty(&tb->owners)) { 200 if (sk->sk_reuse == SK_FORCE_REUSE) 201 goto success; 202 203 if (((tb->fastreuse > 0 && reuse) || 204 (tb->fastreuseport > 0 && 205 !rcu_access_pointer(sk->sk_reuseport_cb) && 206 sk->sk_reuseport && uid_eq(tb->fastuid, uid))) && 207 smallest_size == -1) 208 goto success; 209 if (inet_csk(sk)->icsk_af_ops->bind_conflict(sk, tb, true)) { 210 if ((reuse || 211 (tb->fastreuseport > 0 && 212 sk->sk_reuseport && 213 !rcu_access_pointer(sk->sk_reuseport_cb) && 214 uid_eq(tb->fastuid, uid))) && 215 smallest_size != -1 && --attempts >= 0) { 216 spin_unlock_bh(&head->lock); 217 goto again; 218 } 219 goto fail_unlock; 220 } 221 if (!reuse) 222 tb->fastreuse = 0; 223 if (!sk->sk_reuseport || !uid_eq(tb->fastuid, uid)) 224 tb->fastreuseport = 0; 225 } else { 226 tb->fastreuse = reuse; 227 if (sk->sk_reuseport) { 228 tb->fastreuseport = 1; 229 tb->fastuid = uid; 230 } else { 231 tb->fastreuseport = 0; 232 } 233 } 234 success: 235 if (!inet_csk(sk)->icsk_bind_hash) 236 inet_bind_hash(sk, tb, port); 237 WARN_ON(inet_csk(sk)->icsk_bind_hash != tb); 238 ret = 0; 239 240 fail_unlock: 241 spin_unlock_bh(&head->lock); 242 return ret; 243 } 244 EXPORT_SYMBOL_GPL(inet_csk_get_port); 245 246 /* 247 * Wait for an incoming connection, avoid race conditions. This must be called 248 * with the socket locked. 249 */ 250 static int inet_csk_wait_for_connect(struct sock *sk, long timeo) 251 { 252 struct inet_connection_sock *icsk = inet_csk(sk); 253 DEFINE_WAIT(wait); 254 int err; 255 256 /* 257 * True wake-one mechanism for incoming connections: only 258 * one process gets woken up, not the 'whole herd'. 259 * Since we do not 'race & poll' for established sockets 260 * anymore, the common case will execute the loop only once. 261 * 262 * Subtle issue: "add_wait_queue_exclusive()" will be added 263 * after any current non-exclusive waiters, and we know that 264 * it will always _stay_ after any new non-exclusive waiters 265 * because all non-exclusive waiters are added at the 266 * beginning of the wait-queue. As such, it's ok to "drop" 267 * our exclusiveness temporarily when we get woken up without 268 * having to remove and re-insert us on the wait queue. 269 */ 270 for (;;) { 271 prepare_to_wait_exclusive(sk_sleep(sk), &wait, 272 TASK_INTERRUPTIBLE); 273 release_sock(sk); 274 if (reqsk_queue_empty(&icsk->icsk_accept_queue)) 275 timeo = schedule_timeout(timeo); 276 sched_annotate_sleep(); 277 lock_sock(sk); 278 err = 0; 279 if (!reqsk_queue_empty(&icsk->icsk_accept_queue)) 280 break; 281 err = -EINVAL; 282 if (sk->sk_state != TCP_LISTEN) 283 break; 284 err = sock_intr_errno(timeo); 285 if (signal_pending(current)) 286 break; 287 err = -EAGAIN; 288 if (!timeo) 289 break; 290 } 291 finish_wait(sk_sleep(sk), &wait); 292 return err; 293 } 294 295 /* 296 * This will accept the next outstanding connection. 297 */ 298 struct sock *inet_csk_accept(struct sock *sk, int flags, int *err) 299 { 300 struct inet_connection_sock *icsk = inet_csk(sk); 301 struct request_sock_queue *queue = &icsk->icsk_accept_queue; 302 struct request_sock *req; 303 struct sock *newsk; 304 int error; 305 306 lock_sock(sk); 307 308 /* We need to make sure that this socket is listening, 309 * and that it has something pending. 310 */ 311 error = -EINVAL; 312 if (sk->sk_state != TCP_LISTEN) 313 goto out_err; 314 315 /* Find already established connection */ 316 if (reqsk_queue_empty(queue)) { 317 long timeo = sock_rcvtimeo(sk, flags & O_NONBLOCK); 318 319 /* If this is a non blocking socket don't sleep */ 320 error = -EAGAIN; 321 if (!timeo) 322 goto out_err; 323 324 error = inet_csk_wait_for_connect(sk, timeo); 325 if (error) 326 goto out_err; 327 } 328 req = reqsk_queue_remove(queue, sk); 329 newsk = req->sk; 330 331 if (sk->sk_protocol == IPPROTO_TCP && 332 tcp_rsk(req)->tfo_listener) { 333 spin_lock_bh(&queue->fastopenq.lock); 334 if (tcp_rsk(req)->tfo_listener) { 335 /* We are still waiting for the final ACK from 3WHS 336 * so can't free req now. Instead, we set req->sk to 337 * NULL to signify that the child socket is taken 338 * so reqsk_fastopen_remove() will free the req 339 * when 3WHS finishes (or is aborted). 340 */ 341 req->sk = NULL; 342 req = NULL; 343 } 344 spin_unlock_bh(&queue->fastopenq.lock); 345 } 346 out: 347 release_sock(sk); 348 if (req) 349 reqsk_put(req); 350 return newsk; 351 out_err: 352 newsk = NULL; 353 req = NULL; 354 *err = error; 355 goto out; 356 } 357 EXPORT_SYMBOL(inet_csk_accept); 358 359 /* 360 * Using different timers for retransmit, delayed acks and probes 361 * We may wish use just one timer maintaining a list of expire jiffies 362 * to optimize. 363 */ 364 void inet_csk_init_xmit_timers(struct sock *sk, 365 void (*retransmit_handler)(unsigned long), 366 void (*delack_handler)(unsigned long), 367 void (*keepalive_handler)(unsigned long)) 368 { 369 struct inet_connection_sock *icsk = inet_csk(sk); 370 371 setup_timer(&icsk->icsk_retransmit_timer, retransmit_handler, 372 (unsigned long)sk); 373 setup_timer(&icsk->icsk_delack_timer, delack_handler, 374 (unsigned long)sk); 375 setup_timer(&sk->sk_timer, keepalive_handler, (unsigned long)sk); 376 icsk->icsk_pending = icsk->icsk_ack.pending = 0; 377 } 378 EXPORT_SYMBOL(inet_csk_init_xmit_timers); 379 380 void inet_csk_clear_xmit_timers(struct sock *sk) 381 { 382 struct inet_connection_sock *icsk = inet_csk(sk); 383 384 icsk->icsk_pending = icsk->icsk_ack.pending = icsk->icsk_ack.blocked = 0; 385 386 sk_stop_timer(sk, &icsk->icsk_retransmit_timer); 387 sk_stop_timer(sk, &icsk->icsk_delack_timer); 388 sk_stop_timer(sk, &sk->sk_timer); 389 } 390 EXPORT_SYMBOL(inet_csk_clear_xmit_timers); 391 392 void inet_csk_delete_keepalive_timer(struct sock *sk) 393 { 394 sk_stop_timer(sk, &sk->sk_timer); 395 } 396 EXPORT_SYMBOL(inet_csk_delete_keepalive_timer); 397 398 void inet_csk_reset_keepalive_timer(struct sock *sk, unsigned long len) 399 { 400 sk_reset_timer(sk, &sk->sk_timer, jiffies + len); 401 } 402 EXPORT_SYMBOL(inet_csk_reset_keepalive_timer); 403 404 struct dst_entry *inet_csk_route_req(const struct sock *sk, 405 struct flowi4 *fl4, 406 const struct request_sock *req) 407 { 408 const struct inet_request_sock *ireq = inet_rsk(req); 409 struct net *net = read_pnet(&ireq->ireq_net); 410 struct ip_options_rcu *opt = ireq->opt; 411 struct rtable *rt; 412 413 flowi4_init_output(fl4, ireq->ir_iif, ireq->ir_mark, 414 RT_CONN_FLAGS(sk), RT_SCOPE_UNIVERSE, 415 sk->sk_protocol, inet_sk_flowi_flags(sk), 416 (opt && opt->opt.srr) ? opt->opt.faddr : ireq->ir_rmt_addr, 417 ireq->ir_loc_addr, ireq->ir_rmt_port, 418 htons(ireq->ir_num)); 419 security_req_classify_flow(req, flowi4_to_flowi(fl4)); 420 rt = ip_route_output_flow(net, fl4, sk); 421 if (IS_ERR(rt)) 422 goto no_route; 423 if (opt && opt->opt.is_strictroute && rt->rt_uses_gateway) 424 goto route_err; 425 return &rt->dst; 426 427 route_err: 428 ip_rt_put(rt); 429 no_route: 430 __IP_INC_STATS(net, IPSTATS_MIB_OUTNOROUTES); 431 return NULL; 432 } 433 EXPORT_SYMBOL_GPL(inet_csk_route_req); 434 435 struct dst_entry *inet_csk_route_child_sock(const struct sock *sk, 436 struct sock *newsk, 437 const struct request_sock *req) 438 { 439 const struct inet_request_sock *ireq = inet_rsk(req); 440 struct net *net = read_pnet(&ireq->ireq_net); 441 struct inet_sock *newinet = inet_sk(newsk); 442 struct ip_options_rcu *opt; 443 struct flowi4 *fl4; 444 struct rtable *rt; 445 446 fl4 = &newinet->cork.fl.u.ip4; 447 448 rcu_read_lock(); 449 opt = rcu_dereference(newinet->inet_opt); 450 flowi4_init_output(fl4, ireq->ir_iif, ireq->ir_mark, 451 RT_CONN_FLAGS(sk), RT_SCOPE_UNIVERSE, 452 sk->sk_protocol, inet_sk_flowi_flags(sk), 453 (opt && opt->opt.srr) ? opt->opt.faddr : ireq->ir_rmt_addr, 454 ireq->ir_loc_addr, ireq->ir_rmt_port, 455 htons(ireq->ir_num)); 456 security_req_classify_flow(req, flowi4_to_flowi(fl4)); 457 rt = ip_route_output_flow(net, fl4, sk); 458 if (IS_ERR(rt)) 459 goto no_route; 460 if (opt && opt->opt.is_strictroute && rt->rt_uses_gateway) 461 goto route_err; 462 rcu_read_unlock(); 463 return &rt->dst; 464 465 route_err: 466 ip_rt_put(rt); 467 no_route: 468 rcu_read_unlock(); 469 __IP_INC_STATS(net, IPSTATS_MIB_OUTNOROUTES); 470 return NULL; 471 } 472 EXPORT_SYMBOL_GPL(inet_csk_route_child_sock); 473 474 #if IS_ENABLED(CONFIG_IPV6) 475 #define AF_INET_FAMILY(fam) ((fam) == AF_INET) 476 #else 477 #define AF_INET_FAMILY(fam) true 478 #endif 479 480 /* Decide when to expire the request and when to resend SYN-ACK */ 481 static inline void syn_ack_recalc(struct request_sock *req, const int thresh, 482 const int max_retries, 483 const u8 rskq_defer_accept, 484 int *expire, int *resend) 485 { 486 if (!rskq_defer_accept) { 487 *expire = req->num_timeout >= thresh; 488 *resend = 1; 489 return; 490 } 491 *expire = req->num_timeout >= thresh && 492 (!inet_rsk(req)->acked || req->num_timeout >= max_retries); 493 /* 494 * Do not resend while waiting for data after ACK, 495 * start to resend on end of deferring period to give 496 * last chance for data or ACK to create established socket. 497 */ 498 *resend = !inet_rsk(req)->acked || 499 req->num_timeout >= rskq_defer_accept - 1; 500 } 501 502 int inet_rtx_syn_ack(const struct sock *parent, struct request_sock *req) 503 { 504 int err = req->rsk_ops->rtx_syn_ack(parent, req); 505 506 if (!err) 507 req->num_retrans++; 508 return err; 509 } 510 EXPORT_SYMBOL(inet_rtx_syn_ack); 511 512 /* return true if req was found in the ehash table */ 513 static bool reqsk_queue_unlink(struct request_sock_queue *queue, 514 struct request_sock *req) 515 { 516 struct inet_hashinfo *hashinfo = req_to_sk(req)->sk_prot->h.hashinfo; 517 bool found = false; 518 519 if (sk_hashed(req_to_sk(req))) { 520 spinlock_t *lock = inet_ehash_lockp(hashinfo, req->rsk_hash); 521 522 spin_lock(lock); 523 found = __sk_nulls_del_node_init_rcu(req_to_sk(req)); 524 spin_unlock(lock); 525 } 526 if (timer_pending(&req->rsk_timer) && del_timer_sync(&req->rsk_timer)) 527 reqsk_put(req); 528 return found; 529 } 530 531 void inet_csk_reqsk_queue_drop(struct sock *sk, struct request_sock *req) 532 { 533 if (reqsk_queue_unlink(&inet_csk(sk)->icsk_accept_queue, req)) { 534 reqsk_queue_removed(&inet_csk(sk)->icsk_accept_queue, req); 535 reqsk_put(req); 536 } 537 } 538 EXPORT_SYMBOL(inet_csk_reqsk_queue_drop); 539 540 void inet_csk_reqsk_queue_drop_and_put(struct sock *sk, struct request_sock *req) 541 { 542 inet_csk_reqsk_queue_drop(sk, req); 543 reqsk_put(req); 544 } 545 EXPORT_SYMBOL(inet_csk_reqsk_queue_drop_and_put); 546 547 static void reqsk_timer_handler(unsigned long data) 548 { 549 struct request_sock *req = (struct request_sock *)data; 550 struct sock *sk_listener = req->rsk_listener; 551 struct net *net = sock_net(sk_listener); 552 struct inet_connection_sock *icsk = inet_csk(sk_listener); 553 struct request_sock_queue *queue = &icsk->icsk_accept_queue; 554 int qlen, expire = 0, resend = 0; 555 int max_retries, thresh; 556 u8 defer_accept; 557 558 if (sk_state_load(sk_listener) != TCP_LISTEN) 559 goto drop; 560 561 max_retries = icsk->icsk_syn_retries ? : net->ipv4.sysctl_tcp_synack_retries; 562 thresh = max_retries; 563 /* Normally all the openreqs are young and become mature 564 * (i.e. converted to established socket) for first timeout. 565 * If synack was not acknowledged for 1 second, it means 566 * one of the following things: synack was lost, ack was lost, 567 * rtt is high or nobody planned to ack (i.e. synflood). 568 * When server is a bit loaded, queue is populated with old 569 * open requests, reducing effective size of queue. 570 * When server is well loaded, queue size reduces to zero 571 * after several minutes of work. It is not synflood, 572 * it is normal operation. The solution is pruning 573 * too old entries overriding normal timeout, when 574 * situation becomes dangerous. 575 * 576 * Essentially, we reserve half of room for young 577 * embrions; and abort old ones without pity, if old 578 * ones are about to clog our table. 579 */ 580 qlen = reqsk_queue_len(queue); 581 if ((qlen << 1) > max(8U, sk_listener->sk_max_ack_backlog)) { 582 int young = reqsk_queue_len_young(queue) << 1; 583 584 while (thresh > 2) { 585 if (qlen < young) 586 break; 587 thresh--; 588 young <<= 1; 589 } 590 } 591 defer_accept = READ_ONCE(queue->rskq_defer_accept); 592 if (defer_accept) 593 max_retries = defer_accept; 594 syn_ack_recalc(req, thresh, max_retries, defer_accept, 595 &expire, &resend); 596 req->rsk_ops->syn_ack_timeout(req); 597 if (!expire && 598 (!resend || 599 !inet_rtx_syn_ack(sk_listener, req) || 600 inet_rsk(req)->acked)) { 601 unsigned long timeo; 602 603 if (req->num_timeout++ == 0) 604 atomic_dec(&queue->young); 605 timeo = min(TCP_TIMEOUT_INIT << req->num_timeout, TCP_RTO_MAX); 606 mod_timer(&req->rsk_timer, jiffies + timeo); 607 return; 608 } 609 drop: 610 inet_csk_reqsk_queue_drop_and_put(sk_listener, req); 611 } 612 613 static void reqsk_queue_hash_req(struct request_sock *req, 614 unsigned long timeout) 615 { 616 req->num_retrans = 0; 617 req->num_timeout = 0; 618 req->sk = NULL; 619 620 setup_pinned_timer(&req->rsk_timer, reqsk_timer_handler, 621 (unsigned long)req); 622 mod_timer(&req->rsk_timer, jiffies + timeout); 623 624 inet_ehash_insert(req_to_sk(req), NULL); 625 /* before letting lookups find us, make sure all req fields 626 * are committed to memory and refcnt initialized. 627 */ 628 smp_wmb(); 629 atomic_set(&req->rsk_refcnt, 2 + 1); 630 } 631 632 void inet_csk_reqsk_queue_hash_add(struct sock *sk, struct request_sock *req, 633 unsigned long timeout) 634 { 635 reqsk_queue_hash_req(req, timeout); 636 inet_csk_reqsk_queue_added(sk); 637 } 638 EXPORT_SYMBOL_GPL(inet_csk_reqsk_queue_hash_add); 639 640 /** 641 * inet_csk_clone_lock - clone an inet socket, and lock its clone 642 * @sk: the socket to clone 643 * @req: request_sock 644 * @priority: for allocation (%GFP_KERNEL, %GFP_ATOMIC, etc) 645 * 646 * Caller must unlock socket even in error path (bh_unlock_sock(newsk)) 647 */ 648 struct sock *inet_csk_clone_lock(const struct sock *sk, 649 const struct request_sock *req, 650 const gfp_t priority) 651 { 652 struct sock *newsk = sk_clone_lock(sk, priority); 653 654 if (newsk) { 655 struct inet_connection_sock *newicsk = inet_csk(newsk); 656 657 newsk->sk_state = TCP_SYN_RECV; 658 newicsk->icsk_bind_hash = NULL; 659 660 inet_sk(newsk)->inet_dport = inet_rsk(req)->ir_rmt_port; 661 inet_sk(newsk)->inet_num = inet_rsk(req)->ir_num; 662 inet_sk(newsk)->inet_sport = htons(inet_rsk(req)->ir_num); 663 newsk->sk_write_space = sk_stream_write_space; 664 665 /* listeners have SOCK_RCU_FREE, not the children */ 666 sock_reset_flag(newsk, SOCK_RCU_FREE); 667 668 newsk->sk_mark = inet_rsk(req)->ir_mark; 669 atomic64_set(&newsk->sk_cookie, 670 atomic64_read(&inet_rsk(req)->ir_cookie)); 671 672 newicsk->icsk_retransmits = 0; 673 newicsk->icsk_backoff = 0; 674 newicsk->icsk_probes_out = 0; 675 676 /* Deinitialize accept_queue to trap illegal accesses. */ 677 memset(&newicsk->icsk_accept_queue, 0, sizeof(newicsk->icsk_accept_queue)); 678 679 security_inet_csk_clone(newsk, req); 680 } 681 return newsk; 682 } 683 EXPORT_SYMBOL_GPL(inet_csk_clone_lock); 684 685 /* 686 * At this point, there should be no process reference to this 687 * socket, and thus no user references at all. Therefore we 688 * can assume the socket waitqueue is inactive and nobody will 689 * try to jump onto it. 690 */ 691 void inet_csk_destroy_sock(struct sock *sk) 692 { 693 WARN_ON(sk->sk_state != TCP_CLOSE); 694 WARN_ON(!sock_flag(sk, SOCK_DEAD)); 695 696 /* It cannot be in hash table! */ 697 WARN_ON(!sk_unhashed(sk)); 698 699 /* If it has not 0 inet_sk(sk)->inet_num, it must be bound */ 700 WARN_ON(inet_sk(sk)->inet_num && !inet_csk(sk)->icsk_bind_hash); 701 702 sk->sk_prot->destroy(sk); 703 704 sk_stream_kill_queues(sk); 705 706 xfrm_sk_free_policy(sk); 707 708 sk_refcnt_debug_release(sk); 709 710 local_bh_disable(); 711 percpu_counter_dec(sk->sk_prot->orphan_count); 712 local_bh_enable(); 713 sock_put(sk); 714 } 715 EXPORT_SYMBOL(inet_csk_destroy_sock); 716 717 /* This function allows to force a closure of a socket after the call to 718 * tcp/dccp_create_openreq_child(). 719 */ 720 void inet_csk_prepare_forced_close(struct sock *sk) 721 __releases(&sk->sk_lock.slock) 722 { 723 /* sk_clone_lock locked the socket and set refcnt to 2 */ 724 bh_unlock_sock(sk); 725 sock_put(sk); 726 727 /* The below has to be done to allow calling inet_csk_destroy_sock */ 728 sock_set_flag(sk, SOCK_DEAD); 729 percpu_counter_inc(sk->sk_prot->orphan_count); 730 inet_sk(sk)->inet_num = 0; 731 } 732 EXPORT_SYMBOL(inet_csk_prepare_forced_close); 733 734 int inet_csk_listen_start(struct sock *sk, int backlog) 735 { 736 struct inet_connection_sock *icsk = inet_csk(sk); 737 struct inet_sock *inet = inet_sk(sk); 738 int err = -EADDRINUSE; 739 740 reqsk_queue_alloc(&icsk->icsk_accept_queue); 741 742 sk->sk_max_ack_backlog = backlog; 743 sk->sk_ack_backlog = 0; 744 inet_csk_delack_init(sk); 745 746 /* There is race window here: we announce ourselves listening, 747 * but this transition is still not validated by get_port(). 748 * It is OK, because this socket enters to hash table only 749 * after validation is complete. 750 */ 751 sk_state_store(sk, TCP_LISTEN); 752 if (!sk->sk_prot->get_port(sk, inet->inet_num)) { 753 inet->inet_sport = htons(inet->inet_num); 754 755 sk_dst_reset(sk); 756 err = sk->sk_prot->hash(sk); 757 758 if (likely(!err)) 759 return 0; 760 } 761 762 sk->sk_state = TCP_CLOSE; 763 return err; 764 } 765 EXPORT_SYMBOL_GPL(inet_csk_listen_start); 766 767 static void inet_child_forget(struct sock *sk, struct request_sock *req, 768 struct sock *child) 769 { 770 sk->sk_prot->disconnect(child, O_NONBLOCK); 771 772 sock_orphan(child); 773 774 percpu_counter_inc(sk->sk_prot->orphan_count); 775 776 if (sk->sk_protocol == IPPROTO_TCP && tcp_rsk(req)->tfo_listener) { 777 BUG_ON(tcp_sk(child)->fastopen_rsk != req); 778 BUG_ON(sk != req->rsk_listener); 779 780 /* Paranoid, to prevent race condition if 781 * an inbound pkt destined for child is 782 * blocked by sock lock in tcp_v4_rcv(). 783 * Also to satisfy an assertion in 784 * tcp_v4_destroy_sock(). 785 */ 786 tcp_sk(child)->fastopen_rsk = NULL; 787 } 788 inet_csk_destroy_sock(child); 789 reqsk_put(req); 790 } 791 792 struct sock *inet_csk_reqsk_queue_add(struct sock *sk, 793 struct request_sock *req, 794 struct sock *child) 795 { 796 struct request_sock_queue *queue = &inet_csk(sk)->icsk_accept_queue; 797 798 spin_lock(&queue->rskq_lock); 799 if (unlikely(sk->sk_state != TCP_LISTEN)) { 800 inet_child_forget(sk, req, child); 801 child = NULL; 802 } else { 803 req->sk = child; 804 req->dl_next = NULL; 805 if (queue->rskq_accept_head == NULL) 806 queue->rskq_accept_head = req; 807 else 808 queue->rskq_accept_tail->dl_next = req; 809 queue->rskq_accept_tail = req; 810 sk_acceptq_added(sk); 811 } 812 spin_unlock(&queue->rskq_lock); 813 return child; 814 } 815 EXPORT_SYMBOL(inet_csk_reqsk_queue_add); 816 817 struct sock *inet_csk_complete_hashdance(struct sock *sk, struct sock *child, 818 struct request_sock *req, bool own_req) 819 { 820 if (own_req) { 821 inet_csk_reqsk_queue_drop(sk, req); 822 reqsk_queue_removed(&inet_csk(sk)->icsk_accept_queue, req); 823 if (inet_csk_reqsk_queue_add(sk, req, child)) 824 return child; 825 } 826 /* Too bad, another child took ownership of the request, undo. */ 827 bh_unlock_sock(child); 828 sock_put(child); 829 return NULL; 830 } 831 EXPORT_SYMBOL(inet_csk_complete_hashdance); 832 833 /* 834 * This routine closes sockets which have been at least partially 835 * opened, but not yet accepted. 836 */ 837 void inet_csk_listen_stop(struct sock *sk) 838 { 839 struct inet_connection_sock *icsk = inet_csk(sk); 840 struct request_sock_queue *queue = &icsk->icsk_accept_queue; 841 struct request_sock *next, *req; 842 843 /* Following specs, it would be better either to send FIN 844 * (and enter FIN-WAIT-1, it is normal close) 845 * or to send active reset (abort). 846 * Certainly, it is pretty dangerous while synflood, but it is 847 * bad justification for our negligence 8) 848 * To be honest, we are not able to make either 849 * of the variants now. --ANK 850 */ 851 while ((req = reqsk_queue_remove(queue, sk)) != NULL) { 852 struct sock *child = req->sk; 853 854 local_bh_disable(); 855 bh_lock_sock(child); 856 WARN_ON(sock_owned_by_user(child)); 857 sock_hold(child); 858 859 inet_child_forget(sk, req, child); 860 bh_unlock_sock(child); 861 local_bh_enable(); 862 sock_put(child); 863 864 cond_resched(); 865 } 866 if (queue->fastopenq.rskq_rst_head) { 867 /* Free all the reqs queued in rskq_rst_head. */ 868 spin_lock_bh(&queue->fastopenq.lock); 869 req = queue->fastopenq.rskq_rst_head; 870 queue->fastopenq.rskq_rst_head = NULL; 871 spin_unlock_bh(&queue->fastopenq.lock); 872 while (req != NULL) { 873 next = req->dl_next; 874 reqsk_put(req); 875 req = next; 876 } 877 } 878 WARN_ON_ONCE(sk->sk_ack_backlog); 879 } 880 EXPORT_SYMBOL_GPL(inet_csk_listen_stop); 881 882 void inet_csk_addr2sockaddr(struct sock *sk, struct sockaddr *uaddr) 883 { 884 struct sockaddr_in *sin = (struct sockaddr_in *)uaddr; 885 const struct inet_sock *inet = inet_sk(sk); 886 887 sin->sin_family = AF_INET; 888 sin->sin_addr.s_addr = inet->inet_daddr; 889 sin->sin_port = inet->inet_dport; 890 } 891 EXPORT_SYMBOL_GPL(inet_csk_addr2sockaddr); 892 893 #ifdef CONFIG_COMPAT 894 int inet_csk_compat_getsockopt(struct sock *sk, int level, int optname, 895 char __user *optval, int __user *optlen) 896 { 897 const struct inet_connection_sock *icsk = inet_csk(sk); 898 899 if (icsk->icsk_af_ops->compat_getsockopt) 900 return icsk->icsk_af_ops->compat_getsockopt(sk, level, optname, 901 optval, optlen); 902 return icsk->icsk_af_ops->getsockopt(sk, level, optname, 903 optval, optlen); 904 } 905 EXPORT_SYMBOL_GPL(inet_csk_compat_getsockopt); 906 907 int inet_csk_compat_setsockopt(struct sock *sk, int level, int optname, 908 char __user *optval, unsigned int optlen) 909 { 910 const struct inet_connection_sock *icsk = inet_csk(sk); 911 912 if (icsk->icsk_af_ops->compat_setsockopt) 913 return icsk->icsk_af_ops->compat_setsockopt(sk, level, optname, 914 optval, optlen); 915 return icsk->icsk_af_ops->setsockopt(sk, level, optname, 916 optval, optlen); 917 } 918 EXPORT_SYMBOL_GPL(inet_csk_compat_setsockopt); 919 #endif 920 921 static struct dst_entry *inet_csk_rebuild_route(struct sock *sk, struct flowi *fl) 922 { 923 const struct inet_sock *inet = inet_sk(sk); 924 const struct ip_options_rcu *inet_opt; 925 __be32 daddr = inet->inet_daddr; 926 struct flowi4 *fl4; 927 struct rtable *rt; 928 929 rcu_read_lock(); 930 inet_opt = rcu_dereference(inet->inet_opt); 931 if (inet_opt && inet_opt->opt.srr) 932 daddr = inet_opt->opt.faddr; 933 fl4 = &fl->u.ip4; 934 rt = ip_route_output_ports(sock_net(sk), fl4, sk, daddr, 935 inet->inet_saddr, inet->inet_dport, 936 inet->inet_sport, sk->sk_protocol, 937 RT_CONN_FLAGS(sk), sk->sk_bound_dev_if); 938 if (IS_ERR(rt)) 939 rt = NULL; 940 if (rt) 941 sk_setup_caps(sk, &rt->dst); 942 rcu_read_unlock(); 943 944 return &rt->dst; 945 } 946 947 struct dst_entry *inet_csk_update_pmtu(struct sock *sk, u32 mtu) 948 { 949 struct dst_entry *dst = __sk_dst_check(sk, 0); 950 struct inet_sock *inet = inet_sk(sk); 951 952 if (!dst) { 953 dst = inet_csk_rebuild_route(sk, &inet->cork.fl); 954 if (!dst) 955 goto out; 956 } 957 dst->ops->update_pmtu(dst, sk, NULL, mtu); 958 959 dst = __sk_dst_check(sk, 0); 960 if (!dst) 961 dst = inet_csk_rebuild_route(sk, &inet->cork.fl); 962 out: 963 return dst; 964 } 965 EXPORT_SYMBOL_GPL(inet_csk_update_pmtu); 966