1 // SPDX-License-Identifier: GPL-2.0-only 2 /* Connection state tracking for netfilter. This is separated from, 3 but required by, the NAT layer; it can also be used by an iptables 4 extension. */ 5 6 /* (C) 1999-2001 Paul `Rusty' Russell 7 * (C) 2002-2006 Netfilter Core Team <coreteam@netfilter.org> 8 * (C) 2003,2004 USAGI/WIDE Project <http://www.linux-ipv6.org> 9 * (C) 2005-2012 Patrick McHardy <kaber@trash.net> 10 */ 11 12 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 13 14 #include <linux/types.h> 15 #include <linux/netfilter.h> 16 #include <linux/module.h> 17 #include <linux/sched.h> 18 #include <linux/skbuff.h> 19 #include <linux/proc_fs.h> 20 #include <linux/vmalloc.h> 21 #include <linux/stddef.h> 22 #include <linux/slab.h> 23 #include <linux/random.h> 24 #include <linux/jhash.h> 25 #include <linux/siphash.h> 26 #include <linux/err.h> 27 #include <linux/percpu.h> 28 #include <linux/moduleparam.h> 29 #include <linux/notifier.h> 30 #include <linux/kernel.h> 31 #include <linux/netdevice.h> 32 #include <linux/socket.h> 33 #include <linux/mm.h> 34 #include <linux/nsproxy.h> 35 #include <linux/rculist_nulls.h> 36 37 #include <net/netfilter/nf_conntrack.h> 38 #include <net/netfilter/nf_conntrack_l4proto.h> 39 #include <net/netfilter/nf_conntrack_expect.h> 40 #include <net/netfilter/nf_conntrack_helper.h> 41 #include <net/netfilter/nf_conntrack_seqadj.h> 42 #include <net/netfilter/nf_conntrack_core.h> 43 #include <net/netfilter/nf_conntrack_extend.h> 44 #include <net/netfilter/nf_conntrack_acct.h> 45 #include <net/netfilter/nf_conntrack_ecache.h> 46 #include <net/netfilter/nf_conntrack_zones.h> 47 #include <net/netfilter/nf_conntrack_timestamp.h> 48 #include <net/netfilter/nf_conntrack_timeout.h> 49 #include <net/netfilter/nf_conntrack_labels.h> 50 #include <net/netfilter/nf_conntrack_synproxy.h> 51 #include <net/netfilter/nf_nat.h> 52 #include <net/netfilter/nf_nat_helper.h> 53 #include <net/netns/hash.h> 54 #include <net/ip.h> 55 56 #include "nf_internals.h" 57 58 __cacheline_aligned_in_smp spinlock_t nf_conntrack_locks[CONNTRACK_LOCKS]; 59 EXPORT_SYMBOL_GPL(nf_conntrack_locks); 60 61 __cacheline_aligned_in_smp DEFINE_SPINLOCK(nf_conntrack_expect_lock); 62 EXPORT_SYMBOL_GPL(nf_conntrack_expect_lock); 63 64 struct hlist_nulls_head *nf_conntrack_hash __read_mostly; 65 EXPORT_SYMBOL_GPL(nf_conntrack_hash); 66 67 struct conntrack_gc_work { 68 struct delayed_work dwork; 69 u32 last_bucket; 70 bool exiting; 71 bool early_drop; 72 long next_gc_run; 73 }; 74 75 static __read_mostly struct kmem_cache *nf_conntrack_cachep; 76 static DEFINE_SPINLOCK(nf_conntrack_locks_all_lock); 77 static __read_mostly bool nf_conntrack_locks_all; 78 79 /* every gc cycle scans at most 1/GC_MAX_BUCKETS_DIV part of table */ 80 #define GC_MAX_BUCKETS_DIV 128u 81 /* upper bound of full table scan */ 82 #define GC_MAX_SCAN_JIFFIES (16u * HZ) 83 /* desired ratio of entries found to be expired */ 84 #define GC_EVICT_RATIO 50u 85 86 static struct conntrack_gc_work conntrack_gc_work; 87 88 void nf_conntrack_lock(spinlock_t *lock) __acquires(lock) 89 { 90 /* 1) Acquire the lock */ 91 spin_lock(lock); 92 93 /* 2) read nf_conntrack_locks_all, with ACQUIRE semantics 94 * It pairs with the smp_store_release() in nf_conntrack_all_unlock() 95 */ 96 if (likely(smp_load_acquire(&nf_conntrack_locks_all) == false)) 97 return; 98 99 /* fast path failed, unlock */ 100 spin_unlock(lock); 101 102 /* Slow path 1) get global lock */ 103 spin_lock(&nf_conntrack_locks_all_lock); 104 105 /* Slow path 2) get the lock we want */ 106 spin_lock(lock); 107 108 /* Slow path 3) release the global lock */ 109 spin_unlock(&nf_conntrack_locks_all_lock); 110 } 111 EXPORT_SYMBOL_GPL(nf_conntrack_lock); 112 113 static void nf_conntrack_double_unlock(unsigned int h1, unsigned int h2) 114 { 115 h1 %= CONNTRACK_LOCKS; 116 h2 %= CONNTRACK_LOCKS; 117 spin_unlock(&nf_conntrack_locks[h1]); 118 if (h1 != h2) 119 spin_unlock(&nf_conntrack_locks[h2]); 120 } 121 122 /* return true if we need to recompute hashes (in case hash table was resized) */ 123 static bool nf_conntrack_double_lock(struct net *net, unsigned int h1, 124 unsigned int h2, unsigned int sequence) 125 { 126 h1 %= CONNTRACK_LOCKS; 127 h2 %= CONNTRACK_LOCKS; 128 if (h1 <= h2) { 129 nf_conntrack_lock(&nf_conntrack_locks[h1]); 130 if (h1 != h2) 131 spin_lock_nested(&nf_conntrack_locks[h2], 132 SINGLE_DEPTH_NESTING); 133 } else { 134 nf_conntrack_lock(&nf_conntrack_locks[h2]); 135 spin_lock_nested(&nf_conntrack_locks[h1], 136 SINGLE_DEPTH_NESTING); 137 } 138 if (read_seqcount_retry(&nf_conntrack_generation, sequence)) { 139 nf_conntrack_double_unlock(h1, h2); 140 return true; 141 } 142 return false; 143 } 144 145 static void nf_conntrack_all_lock(void) 146 __acquires(&nf_conntrack_locks_all_lock) 147 { 148 int i; 149 150 spin_lock(&nf_conntrack_locks_all_lock); 151 152 nf_conntrack_locks_all = true; 153 154 for (i = 0; i < CONNTRACK_LOCKS; i++) { 155 spin_lock(&nf_conntrack_locks[i]); 156 157 /* This spin_unlock provides the "release" to ensure that 158 * nf_conntrack_locks_all==true is visible to everyone that 159 * acquired spin_lock(&nf_conntrack_locks[]). 160 */ 161 spin_unlock(&nf_conntrack_locks[i]); 162 } 163 } 164 165 static void nf_conntrack_all_unlock(void) 166 __releases(&nf_conntrack_locks_all_lock) 167 { 168 /* All prior stores must be complete before we clear 169 * 'nf_conntrack_locks_all'. Otherwise nf_conntrack_lock() 170 * might observe the false value but not the entire 171 * critical section. 172 * It pairs with the smp_load_acquire() in nf_conntrack_lock() 173 */ 174 smp_store_release(&nf_conntrack_locks_all, false); 175 spin_unlock(&nf_conntrack_locks_all_lock); 176 } 177 178 unsigned int nf_conntrack_htable_size __read_mostly; 179 EXPORT_SYMBOL_GPL(nf_conntrack_htable_size); 180 181 unsigned int nf_conntrack_max __read_mostly; 182 EXPORT_SYMBOL_GPL(nf_conntrack_max); 183 seqcount_spinlock_t nf_conntrack_generation __read_mostly; 184 static unsigned int nf_conntrack_hash_rnd __read_mostly; 185 186 static u32 hash_conntrack_raw(const struct nf_conntrack_tuple *tuple, 187 const struct net *net) 188 { 189 unsigned int n; 190 u32 seed; 191 192 get_random_once(&nf_conntrack_hash_rnd, sizeof(nf_conntrack_hash_rnd)); 193 194 /* The direction must be ignored, so we hash everything up to the 195 * destination ports (which is a multiple of 4) and treat the last 196 * three bytes manually. 197 */ 198 seed = nf_conntrack_hash_rnd ^ net_hash_mix(net); 199 n = (sizeof(tuple->src) + sizeof(tuple->dst.u3)) / sizeof(u32); 200 return jhash2((u32 *)tuple, n, seed ^ 201 (((__force __u16)tuple->dst.u.all << 16) | 202 tuple->dst.protonum)); 203 } 204 205 static u32 scale_hash(u32 hash) 206 { 207 return reciprocal_scale(hash, nf_conntrack_htable_size); 208 } 209 210 static u32 __hash_conntrack(const struct net *net, 211 const struct nf_conntrack_tuple *tuple, 212 unsigned int size) 213 { 214 return reciprocal_scale(hash_conntrack_raw(tuple, net), size); 215 } 216 217 static u32 hash_conntrack(const struct net *net, 218 const struct nf_conntrack_tuple *tuple) 219 { 220 return scale_hash(hash_conntrack_raw(tuple, net)); 221 } 222 223 static bool nf_ct_get_tuple_ports(const struct sk_buff *skb, 224 unsigned int dataoff, 225 struct nf_conntrack_tuple *tuple) 226 { struct { 227 __be16 sport; 228 __be16 dport; 229 } _inet_hdr, *inet_hdr; 230 231 /* Actually only need first 4 bytes to get ports. */ 232 inet_hdr = skb_header_pointer(skb, dataoff, sizeof(_inet_hdr), &_inet_hdr); 233 if (!inet_hdr) 234 return false; 235 236 tuple->src.u.udp.port = inet_hdr->sport; 237 tuple->dst.u.udp.port = inet_hdr->dport; 238 return true; 239 } 240 241 static bool 242 nf_ct_get_tuple(const struct sk_buff *skb, 243 unsigned int nhoff, 244 unsigned int dataoff, 245 u_int16_t l3num, 246 u_int8_t protonum, 247 struct net *net, 248 struct nf_conntrack_tuple *tuple) 249 { 250 unsigned int size; 251 const __be32 *ap; 252 __be32 _addrs[8]; 253 254 memset(tuple, 0, sizeof(*tuple)); 255 256 tuple->src.l3num = l3num; 257 switch (l3num) { 258 case NFPROTO_IPV4: 259 nhoff += offsetof(struct iphdr, saddr); 260 size = 2 * sizeof(__be32); 261 break; 262 case NFPROTO_IPV6: 263 nhoff += offsetof(struct ipv6hdr, saddr); 264 size = sizeof(_addrs); 265 break; 266 default: 267 return true; 268 } 269 270 ap = skb_header_pointer(skb, nhoff, size, _addrs); 271 if (!ap) 272 return false; 273 274 switch (l3num) { 275 case NFPROTO_IPV4: 276 tuple->src.u3.ip = ap[0]; 277 tuple->dst.u3.ip = ap[1]; 278 break; 279 case NFPROTO_IPV6: 280 memcpy(tuple->src.u3.ip6, ap, sizeof(tuple->src.u3.ip6)); 281 memcpy(tuple->dst.u3.ip6, ap + 4, sizeof(tuple->dst.u3.ip6)); 282 break; 283 } 284 285 tuple->dst.protonum = protonum; 286 tuple->dst.dir = IP_CT_DIR_ORIGINAL; 287 288 switch (protonum) { 289 #if IS_ENABLED(CONFIG_IPV6) 290 case IPPROTO_ICMPV6: 291 return icmpv6_pkt_to_tuple(skb, dataoff, net, tuple); 292 #endif 293 case IPPROTO_ICMP: 294 return icmp_pkt_to_tuple(skb, dataoff, net, tuple); 295 #ifdef CONFIG_NF_CT_PROTO_GRE 296 case IPPROTO_GRE: 297 return gre_pkt_to_tuple(skb, dataoff, net, tuple); 298 #endif 299 case IPPROTO_TCP: 300 case IPPROTO_UDP: /* fallthrough */ 301 return nf_ct_get_tuple_ports(skb, dataoff, tuple); 302 #ifdef CONFIG_NF_CT_PROTO_UDPLITE 303 case IPPROTO_UDPLITE: 304 return nf_ct_get_tuple_ports(skb, dataoff, tuple); 305 #endif 306 #ifdef CONFIG_NF_CT_PROTO_SCTP 307 case IPPROTO_SCTP: 308 return nf_ct_get_tuple_ports(skb, dataoff, tuple); 309 #endif 310 #ifdef CONFIG_NF_CT_PROTO_DCCP 311 case IPPROTO_DCCP: 312 return nf_ct_get_tuple_ports(skb, dataoff, tuple); 313 #endif 314 default: 315 break; 316 } 317 318 return true; 319 } 320 321 static int ipv4_get_l4proto(const struct sk_buff *skb, unsigned int nhoff, 322 u_int8_t *protonum) 323 { 324 int dataoff = -1; 325 const struct iphdr *iph; 326 struct iphdr _iph; 327 328 iph = skb_header_pointer(skb, nhoff, sizeof(_iph), &_iph); 329 if (!iph) 330 return -1; 331 332 /* Conntrack defragments packets, we might still see fragments 333 * inside ICMP packets though. 334 */ 335 if (iph->frag_off & htons(IP_OFFSET)) 336 return -1; 337 338 dataoff = nhoff + (iph->ihl << 2); 339 *protonum = iph->protocol; 340 341 /* Check bogus IP headers */ 342 if (dataoff > skb->len) { 343 pr_debug("bogus IPv4 packet: nhoff %u, ihl %u, skblen %u\n", 344 nhoff, iph->ihl << 2, skb->len); 345 return -1; 346 } 347 return dataoff; 348 } 349 350 #if IS_ENABLED(CONFIG_IPV6) 351 static int ipv6_get_l4proto(const struct sk_buff *skb, unsigned int nhoff, 352 u8 *protonum) 353 { 354 int protoff = -1; 355 unsigned int extoff = nhoff + sizeof(struct ipv6hdr); 356 __be16 frag_off; 357 u8 nexthdr; 358 359 if (skb_copy_bits(skb, nhoff + offsetof(struct ipv6hdr, nexthdr), 360 &nexthdr, sizeof(nexthdr)) != 0) { 361 pr_debug("can't get nexthdr\n"); 362 return -1; 363 } 364 protoff = ipv6_skip_exthdr(skb, extoff, &nexthdr, &frag_off); 365 /* 366 * (protoff == skb->len) means the packet has not data, just 367 * IPv6 and possibly extensions headers, but it is tracked anyway 368 */ 369 if (protoff < 0 || (frag_off & htons(~0x7)) != 0) { 370 pr_debug("can't find proto in pkt\n"); 371 return -1; 372 } 373 374 *protonum = nexthdr; 375 return protoff; 376 } 377 #endif 378 379 static int get_l4proto(const struct sk_buff *skb, 380 unsigned int nhoff, u8 pf, u8 *l4num) 381 { 382 switch (pf) { 383 case NFPROTO_IPV4: 384 return ipv4_get_l4proto(skb, nhoff, l4num); 385 #if IS_ENABLED(CONFIG_IPV6) 386 case NFPROTO_IPV6: 387 return ipv6_get_l4proto(skb, nhoff, l4num); 388 #endif 389 default: 390 *l4num = 0; 391 break; 392 } 393 return -1; 394 } 395 396 bool nf_ct_get_tuplepr(const struct sk_buff *skb, unsigned int nhoff, 397 u_int16_t l3num, 398 struct net *net, struct nf_conntrack_tuple *tuple) 399 { 400 u8 protonum; 401 int protoff; 402 403 protoff = get_l4proto(skb, nhoff, l3num, &protonum); 404 if (protoff <= 0) 405 return false; 406 407 return nf_ct_get_tuple(skb, nhoff, protoff, l3num, protonum, net, tuple); 408 } 409 EXPORT_SYMBOL_GPL(nf_ct_get_tuplepr); 410 411 bool 412 nf_ct_invert_tuple(struct nf_conntrack_tuple *inverse, 413 const struct nf_conntrack_tuple *orig) 414 { 415 memset(inverse, 0, sizeof(*inverse)); 416 417 inverse->src.l3num = orig->src.l3num; 418 419 switch (orig->src.l3num) { 420 case NFPROTO_IPV4: 421 inverse->src.u3.ip = orig->dst.u3.ip; 422 inverse->dst.u3.ip = orig->src.u3.ip; 423 break; 424 case NFPROTO_IPV6: 425 inverse->src.u3.in6 = orig->dst.u3.in6; 426 inverse->dst.u3.in6 = orig->src.u3.in6; 427 break; 428 default: 429 break; 430 } 431 432 inverse->dst.dir = !orig->dst.dir; 433 434 inverse->dst.protonum = orig->dst.protonum; 435 436 switch (orig->dst.protonum) { 437 case IPPROTO_ICMP: 438 return nf_conntrack_invert_icmp_tuple(inverse, orig); 439 #if IS_ENABLED(CONFIG_IPV6) 440 case IPPROTO_ICMPV6: 441 return nf_conntrack_invert_icmpv6_tuple(inverse, orig); 442 #endif 443 } 444 445 inverse->src.u.all = orig->dst.u.all; 446 inverse->dst.u.all = orig->src.u.all; 447 return true; 448 } 449 EXPORT_SYMBOL_GPL(nf_ct_invert_tuple); 450 451 /* Generate a almost-unique pseudo-id for a given conntrack. 452 * 453 * intentionally doesn't re-use any of the seeds used for hash 454 * table location, we assume id gets exposed to userspace. 455 * 456 * Following nf_conn items do not change throughout lifetime 457 * of the nf_conn: 458 * 459 * 1. nf_conn address 460 * 2. nf_conn->master address (normally NULL) 461 * 3. the associated net namespace 462 * 4. the original direction tuple 463 */ 464 u32 nf_ct_get_id(const struct nf_conn *ct) 465 { 466 static __read_mostly siphash_key_t ct_id_seed; 467 unsigned long a, b, c, d; 468 469 net_get_random_once(&ct_id_seed, sizeof(ct_id_seed)); 470 471 a = (unsigned long)ct; 472 b = (unsigned long)ct->master; 473 c = (unsigned long)nf_ct_net(ct); 474 d = (unsigned long)siphash(&ct->tuplehash[IP_CT_DIR_ORIGINAL].tuple, 475 sizeof(ct->tuplehash[IP_CT_DIR_ORIGINAL].tuple), 476 &ct_id_seed); 477 #ifdef CONFIG_64BIT 478 return siphash_4u64((u64)a, (u64)b, (u64)c, (u64)d, &ct_id_seed); 479 #else 480 return siphash_4u32((u32)a, (u32)b, (u32)c, (u32)d, &ct_id_seed); 481 #endif 482 } 483 EXPORT_SYMBOL_GPL(nf_ct_get_id); 484 485 static void 486 clean_from_lists(struct nf_conn *ct) 487 { 488 pr_debug("clean_from_lists(%p)\n", ct); 489 hlist_nulls_del_rcu(&ct->tuplehash[IP_CT_DIR_ORIGINAL].hnnode); 490 hlist_nulls_del_rcu(&ct->tuplehash[IP_CT_DIR_REPLY].hnnode); 491 492 /* Destroy all pending expectations */ 493 nf_ct_remove_expectations(ct); 494 } 495 496 /* must be called with local_bh_disable */ 497 static void nf_ct_add_to_dying_list(struct nf_conn *ct) 498 { 499 struct ct_pcpu *pcpu; 500 501 /* add this conntrack to the (per cpu) dying list */ 502 ct->cpu = smp_processor_id(); 503 pcpu = per_cpu_ptr(nf_ct_net(ct)->ct.pcpu_lists, ct->cpu); 504 505 spin_lock(&pcpu->lock); 506 hlist_nulls_add_head(&ct->tuplehash[IP_CT_DIR_ORIGINAL].hnnode, 507 &pcpu->dying); 508 spin_unlock(&pcpu->lock); 509 } 510 511 /* must be called with local_bh_disable */ 512 static void nf_ct_add_to_unconfirmed_list(struct nf_conn *ct) 513 { 514 struct ct_pcpu *pcpu; 515 516 /* add this conntrack to the (per cpu) unconfirmed list */ 517 ct->cpu = smp_processor_id(); 518 pcpu = per_cpu_ptr(nf_ct_net(ct)->ct.pcpu_lists, ct->cpu); 519 520 spin_lock(&pcpu->lock); 521 hlist_nulls_add_head(&ct->tuplehash[IP_CT_DIR_ORIGINAL].hnnode, 522 &pcpu->unconfirmed); 523 spin_unlock(&pcpu->lock); 524 } 525 526 /* must be called with local_bh_disable */ 527 static void nf_ct_del_from_dying_or_unconfirmed_list(struct nf_conn *ct) 528 { 529 struct ct_pcpu *pcpu; 530 531 /* We overload first tuple to link into unconfirmed or dying list.*/ 532 pcpu = per_cpu_ptr(nf_ct_net(ct)->ct.pcpu_lists, ct->cpu); 533 534 spin_lock(&pcpu->lock); 535 BUG_ON(hlist_nulls_unhashed(&ct->tuplehash[IP_CT_DIR_ORIGINAL].hnnode)); 536 hlist_nulls_del_rcu(&ct->tuplehash[IP_CT_DIR_ORIGINAL].hnnode); 537 spin_unlock(&pcpu->lock); 538 } 539 540 #define NFCT_ALIGN(len) (((len) + NFCT_INFOMASK) & ~NFCT_INFOMASK) 541 542 /* Released via destroy_conntrack() */ 543 struct nf_conn *nf_ct_tmpl_alloc(struct net *net, 544 const struct nf_conntrack_zone *zone, 545 gfp_t flags) 546 { 547 struct nf_conn *tmpl, *p; 548 549 if (ARCH_KMALLOC_MINALIGN <= NFCT_INFOMASK) { 550 tmpl = kzalloc(sizeof(*tmpl) + NFCT_INFOMASK, flags); 551 if (!tmpl) 552 return NULL; 553 554 p = tmpl; 555 tmpl = (struct nf_conn *)NFCT_ALIGN((unsigned long)p); 556 if (tmpl != p) { 557 tmpl = (struct nf_conn *)NFCT_ALIGN((unsigned long)p); 558 tmpl->proto.tmpl_padto = (char *)tmpl - (char *)p; 559 } 560 } else { 561 tmpl = kzalloc(sizeof(*tmpl), flags); 562 if (!tmpl) 563 return NULL; 564 } 565 566 tmpl->status = IPS_TEMPLATE; 567 write_pnet(&tmpl->ct_net, net); 568 nf_ct_zone_add(tmpl, zone); 569 atomic_set(&tmpl->ct_general.use, 0); 570 571 return tmpl; 572 } 573 EXPORT_SYMBOL_GPL(nf_ct_tmpl_alloc); 574 575 void nf_ct_tmpl_free(struct nf_conn *tmpl) 576 { 577 nf_ct_ext_destroy(tmpl); 578 579 if (ARCH_KMALLOC_MINALIGN <= NFCT_INFOMASK) 580 kfree((char *)tmpl - tmpl->proto.tmpl_padto); 581 else 582 kfree(tmpl); 583 } 584 EXPORT_SYMBOL_GPL(nf_ct_tmpl_free); 585 586 static void destroy_gre_conntrack(struct nf_conn *ct) 587 { 588 #ifdef CONFIG_NF_CT_PROTO_GRE 589 struct nf_conn *master = ct->master; 590 591 if (master) 592 nf_ct_gre_keymap_destroy(master); 593 #endif 594 } 595 596 static void 597 destroy_conntrack(struct nf_conntrack *nfct) 598 { 599 struct nf_conn *ct = (struct nf_conn *)nfct; 600 601 pr_debug("destroy_conntrack(%p)\n", ct); 602 WARN_ON(atomic_read(&nfct->use) != 0); 603 604 if (unlikely(nf_ct_is_template(ct))) { 605 nf_ct_tmpl_free(ct); 606 return; 607 } 608 609 if (unlikely(nf_ct_protonum(ct) == IPPROTO_GRE)) 610 destroy_gre_conntrack(ct); 611 612 local_bh_disable(); 613 /* Expectations will have been removed in clean_from_lists, 614 * except TFTP can create an expectation on the first packet, 615 * before connection is in the list, so we need to clean here, 616 * too. 617 */ 618 nf_ct_remove_expectations(ct); 619 620 nf_ct_del_from_dying_or_unconfirmed_list(ct); 621 622 local_bh_enable(); 623 624 if (ct->master) 625 nf_ct_put(ct->master); 626 627 pr_debug("destroy_conntrack: returning ct=%p to slab\n", ct); 628 nf_conntrack_free(ct); 629 } 630 631 static void nf_ct_delete_from_lists(struct nf_conn *ct) 632 { 633 struct net *net = nf_ct_net(ct); 634 unsigned int hash, reply_hash; 635 unsigned int sequence; 636 637 nf_ct_helper_destroy(ct); 638 639 local_bh_disable(); 640 do { 641 sequence = read_seqcount_begin(&nf_conntrack_generation); 642 hash = hash_conntrack(net, 643 &ct->tuplehash[IP_CT_DIR_ORIGINAL].tuple); 644 reply_hash = hash_conntrack(net, 645 &ct->tuplehash[IP_CT_DIR_REPLY].tuple); 646 } while (nf_conntrack_double_lock(net, hash, reply_hash, sequence)); 647 648 clean_from_lists(ct); 649 nf_conntrack_double_unlock(hash, reply_hash); 650 651 nf_ct_add_to_dying_list(ct); 652 653 local_bh_enable(); 654 } 655 656 bool nf_ct_delete(struct nf_conn *ct, u32 portid, int report) 657 { 658 struct nf_conn_tstamp *tstamp; 659 660 if (test_and_set_bit(IPS_DYING_BIT, &ct->status)) 661 return false; 662 663 tstamp = nf_conn_tstamp_find(ct); 664 if (tstamp && tstamp->stop == 0) 665 tstamp->stop = ktime_get_real_ns(); 666 667 if (nf_conntrack_event_report(IPCT_DESTROY, ct, 668 portid, report) < 0) { 669 /* destroy event was not delivered. nf_ct_put will 670 * be done by event cache worker on redelivery. 671 */ 672 nf_ct_delete_from_lists(ct); 673 nf_conntrack_ecache_delayed_work(nf_ct_net(ct)); 674 return false; 675 } 676 677 nf_conntrack_ecache_work(nf_ct_net(ct)); 678 nf_ct_delete_from_lists(ct); 679 nf_ct_put(ct); 680 return true; 681 } 682 EXPORT_SYMBOL_GPL(nf_ct_delete); 683 684 static inline bool 685 nf_ct_key_equal(struct nf_conntrack_tuple_hash *h, 686 const struct nf_conntrack_tuple *tuple, 687 const struct nf_conntrack_zone *zone, 688 const struct net *net) 689 { 690 struct nf_conn *ct = nf_ct_tuplehash_to_ctrack(h); 691 692 /* A conntrack can be recreated with the equal tuple, 693 * so we need to check that the conntrack is confirmed 694 */ 695 return nf_ct_tuple_equal(tuple, &h->tuple) && 696 nf_ct_zone_equal(ct, zone, NF_CT_DIRECTION(h)) && 697 nf_ct_is_confirmed(ct) && 698 net_eq(net, nf_ct_net(ct)); 699 } 700 701 static inline bool 702 nf_ct_match(const struct nf_conn *ct1, const struct nf_conn *ct2) 703 { 704 return nf_ct_tuple_equal(&ct1->tuplehash[IP_CT_DIR_ORIGINAL].tuple, 705 &ct2->tuplehash[IP_CT_DIR_ORIGINAL].tuple) && 706 nf_ct_tuple_equal(&ct1->tuplehash[IP_CT_DIR_REPLY].tuple, 707 &ct2->tuplehash[IP_CT_DIR_REPLY].tuple) && 708 nf_ct_zone_equal(ct1, nf_ct_zone(ct2), IP_CT_DIR_ORIGINAL) && 709 nf_ct_zone_equal(ct1, nf_ct_zone(ct2), IP_CT_DIR_REPLY) && 710 net_eq(nf_ct_net(ct1), nf_ct_net(ct2)); 711 } 712 713 /* caller must hold rcu readlock and none of the nf_conntrack_locks */ 714 static void nf_ct_gc_expired(struct nf_conn *ct) 715 { 716 if (!atomic_inc_not_zero(&ct->ct_general.use)) 717 return; 718 719 if (nf_ct_should_gc(ct)) 720 nf_ct_kill(ct); 721 722 nf_ct_put(ct); 723 } 724 725 /* 726 * Warning : 727 * - Caller must take a reference on returned object 728 * and recheck nf_ct_tuple_equal(tuple, &h->tuple) 729 */ 730 static struct nf_conntrack_tuple_hash * 731 ____nf_conntrack_find(struct net *net, const struct nf_conntrack_zone *zone, 732 const struct nf_conntrack_tuple *tuple, u32 hash) 733 { 734 struct nf_conntrack_tuple_hash *h; 735 struct hlist_nulls_head *ct_hash; 736 struct hlist_nulls_node *n; 737 unsigned int bucket, hsize; 738 739 begin: 740 nf_conntrack_get_ht(&ct_hash, &hsize); 741 bucket = reciprocal_scale(hash, hsize); 742 743 hlist_nulls_for_each_entry_rcu(h, n, &ct_hash[bucket], hnnode) { 744 struct nf_conn *ct; 745 746 ct = nf_ct_tuplehash_to_ctrack(h); 747 if (nf_ct_is_expired(ct)) { 748 nf_ct_gc_expired(ct); 749 continue; 750 } 751 752 if (nf_ct_key_equal(h, tuple, zone, net)) 753 return h; 754 } 755 /* 756 * if the nulls value we got at the end of this lookup is 757 * not the expected one, we must restart lookup. 758 * We probably met an item that was moved to another chain. 759 */ 760 if (get_nulls_value(n) != bucket) { 761 NF_CT_STAT_INC_ATOMIC(net, search_restart); 762 goto begin; 763 } 764 765 return NULL; 766 } 767 768 /* Find a connection corresponding to a tuple. */ 769 static struct nf_conntrack_tuple_hash * 770 __nf_conntrack_find_get(struct net *net, const struct nf_conntrack_zone *zone, 771 const struct nf_conntrack_tuple *tuple, u32 hash) 772 { 773 struct nf_conntrack_tuple_hash *h; 774 struct nf_conn *ct; 775 776 rcu_read_lock(); 777 778 h = ____nf_conntrack_find(net, zone, tuple, hash); 779 if (h) { 780 /* We have a candidate that matches the tuple we're interested 781 * in, try to obtain a reference and re-check tuple 782 */ 783 ct = nf_ct_tuplehash_to_ctrack(h); 784 if (likely(atomic_inc_not_zero(&ct->ct_general.use))) { 785 if (likely(nf_ct_key_equal(h, tuple, zone, net))) 786 goto found; 787 788 /* TYPESAFE_BY_RCU recycled the candidate */ 789 nf_ct_put(ct); 790 } 791 792 h = NULL; 793 } 794 found: 795 rcu_read_unlock(); 796 797 return h; 798 } 799 800 struct nf_conntrack_tuple_hash * 801 nf_conntrack_find_get(struct net *net, const struct nf_conntrack_zone *zone, 802 const struct nf_conntrack_tuple *tuple) 803 { 804 return __nf_conntrack_find_get(net, zone, tuple, 805 hash_conntrack_raw(tuple, net)); 806 } 807 EXPORT_SYMBOL_GPL(nf_conntrack_find_get); 808 809 static void __nf_conntrack_hash_insert(struct nf_conn *ct, 810 unsigned int hash, 811 unsigned int reply_hash) 812 { 813 hlist_nulls_add_head_rcu(&ct->tuplehash[IP_CT_DIR_ORIGINAL].hnnode, 814 &nf_conntrack_hash[hash]); 815 hlist_nulls_add_head_rcu(&ct->tuplehash[IP_CT_DIR_REPLY].hnnode, 816 &nf_conntrack_hash[reply_hash]); 817 } 818 819 int 820 nf_conntrack_hash_check_insert(struct nf_conn *ct) 821 { 822 const struct nf_conntrack_zone *zone; 823 struct net *net = nf_ct_net(ct); 824 unsigned int hash, reply_hash; 825 struct nf_conntrack_tuple_hash *h; 826 struct hlist_nulls_node *n; 827 unsigned int sequence; 828 829 zone = nf_ct_zone(ct); 830 831 local_bh_disable(); 832 do { 833 sequence = read_seqcount_begin(&nf_conntrack_generation); 834 hash = hash_conntrack(net, 835 &ct->tuplehash[IP_CT_DIR_ORIGINAL].tuple); 836 reply_hash = hash_conntrack(net, 837 &ct->tuplehash[IP_CT_DIR_REPLY].tuple); 838 } while (nf_conntrack_double_lock(net, hash, reply_hash, sequence)); 839 840 /* See if there's one in the list already, including reverse */ 841 hlist_nulls_for_each_entry(h, n, &nf_conntrack_hash[hash], hnnode) 842 if (nf_ct_key_equal(h, &ct->tuplehash[IP_CT_DIR_ORIGINAL].tuple, 843 zone, net)) 844 goto out; 845 846 hlist_nulls_for_each_entry(h, n, &nf_conntrack_hash[reply_hash], hnnode) 847 if (nf_ct_key_equal(h, &ct->tuplehash[IP_CT_DIR_REPLY].tuple, 848 zone, net)) 849 goto out; 850 851 smp_wmb(); 852 /* The caller holds a reference to this object */ 853 atomic_set(&ct->ct_general.use, 2); 854 __nf_conntrack_hash_insert(ct, hash, reply_hash); 855 nf_conntrack_double_unlock(hash, reply_hash); 856 NF_CT_STAT_INC(net, insert); 857 local_bh_enable(); 858 return 0; 859 860 out: 861 nf_conntrack_double_unlock(hash, reply_hash); 862 NF_CT_STAT_INC(net, insert_failed); 863 local_bh_enable(); 864 return -EEXIST; 865 } 866 EXPORT_SYMBOL_GPL(nf_conntrack_hash_check_insert); 867 868 void nf_ct_acct_add(struct nf_conn *ct, u32 dir, unsigned int packets, 869 unsigned int bytes) 870 { 871 struct nf_conn_acct *acct; 872 873 acct = nf_conn_acct_find(ct); 874 if (acct) { 875 struct nf_conn_counter *counter = acct->counter; 876 877 atomic64_add(packets, &counter[dir].packets); 878 atomic64_add(bytes, &counter[dir].bytes); 879 } 880 } 881 EXPORT_SYMBOL_GPL(nf_ct_acct_add); 882 883 static void nf_ct_acct_merge(struct nf_conn *ct, enum ip_conntrack_info ctinfo, 884 const struct nf_conn *loser_ct) 885 { 886 struct nf_conn_acct *acct; 887 888 acct = nf_conn_acct_find(loser_ct); 889 if (acct) { 890 struct nf_conn_counter *counter = acct->counter; 891 unsigned int bytes; 892 893 /* u32 should be fine since we must have seen one packet. */ 894 bytes = atomic64_read(&counter[CTINFO2DIR(ctinfo)].bytes); 895 nf_ct_acct_update(ct, CTINFO2DIR(ctinfo), bytes); 896 } 897 } 898 899 static void __nf_conntrack_insert_prepare(struct nf_conn *ct) 900 { 901 struct nf_conn_tstamp *tstamp; 902 903 atomic_inc(&ct->ct_general.use); 904 ct->status |= IPS_CONFIRMED; 905 906 /* set conntrack timestamp, if enabled. */ 907 tstamp = nf_conn_tstamp_find(ct); 908 if (tstamp) 909 tstamp->start = ktime_get_real_ns(); 910 } 911 912 static int __nf_ct_resolve_clash(struct sk_buff *skb, 913 struct nf_conntrack_tuple_hash *h) 914 { 915 /* This is the conntrack entry already in hashes that won race. */ 916 struct nf_conn *ct = nf_ct_tuplehash_to_ctrack(h); 917 enum ip_conntrack_info ctinfo; 918 struct nf_conn *loser_ct; 919 920 loser_ct = nf_ct_get(skb, &ctinfo); 921 922 if (nf_ct_is_dying(ct)) 923 return NF_DROP; 924 925 if (!atomic_inc_not_zero(&ct->ct_general.use)) 926 return NF_DROP; 927 928 if (((ct->status & IPS_NAT_DONE_MASK) == 0) || 929 nf_ct_match(ct, loser_ct)) { 930 struct net *net = nf_ct_net(ct); 931 932 nf_ct_acct_merge(ct, ctinfo, loser_ct); 933 nf_ct_add_to_dying_list(loser_ct); 934 nf_conntrack_put(&loser_ct->ct_general); 935 nf_ct_set(skb, ct, ctinfo); 936 937 NF_CT_STAT_INC(net, insert_failed); 938 return NF_ACCEPT; 939 } 940 941 nf_ct_put(ct); 942 return NF_DROP; 943 } 944 945 /** 946 * nf_ct_resolve_clash_harder - attempt to insert clashing conntrack entry 947 * 948 * @skb: skb that causes the collision 949 * @repl_idx: hash slot for reply direction 950 * 951 * Called when origin or reply direction had a clash. 952 * The skb can be handled without packet drop provided the reply direction 953 * is unique or there the existing entry has the identical tuple in both 954 * directions. 955 * 956 * Caller must hold conntrack table locks to prevent concurrent updates. 957 * 958 * Returns NF_DROP if the clash could not be handled. 959 */ 960 static int nf_ct_resolve_clash_harder(struct sk_buff *skb, u32 repl_idx) 961 { 962 struct nf_conn *loser_ct = (struct nf_conn *)skb_nfct(skb); 963 const struct nf_conntrack_zone *zone; 964 struct nf_conntrack_tuple_hash *h; 965 struct hlist_nulls_node *n; 966 struct net *net; 967 968 zone = nf_ct_zone(loser_ct); 969 net = nf_ct_net(loser_ct); 970 971 /* Reply direction must never result in a clash, unless both origin 972 * and reply tuples are identical. 973 */ 974 hlist_nulls_for_each_entry(h, n, &nf_conntrack_hash[repl_idx], hnnode) { 975 if (nf_ct_key_equal(h, 976 &loser_ct->tuplehash[IP_CT_DIR_REPLY].tuple, 977 zone, net)) 978 return __nf_ct_resolve_clash(skb, h); 979 } 980 981 /* We want the clashing entry to go away real soon: 1 second timeout. */ 982 loser_ct->timeout = nfct_time_stamp + HZ; 983 984 /* IPS_NAT_CLASH removes the entry automatically on the first 985 * reply. Also prevents UDP tracker from moving the entry to 986 * ASSURED state, i.e. the entry can always be evicted under 987 * pressure. 988 */ 989 loser_ct->status |= IPS_FIXED_TIMEOUT | IPS_NAT_CLASH; 990 991 __nf_conntrack_insert_prepare(loser_ct); 992 993 /* fake add for ORIGINAL dir: we want lookups to only find the entry 994 * already in the table. This also hides the clashing entry from 995 * ctnetlink iteration, i.e. conntrack -L won't show them. 996 */ 997 hlist_nulls_add_fake(&loser_ct->tuplehash[IP_CT_DIR_ORIGINAL].hnnode); 998 999 hlist_nulls_add_head_rcu(&loser_ct->tuplehash[IP_CT_DIR_REPLY].hnnode, 1000 &nf_conntrack_hash[repl_idx]); 1001 return NF_ACCEPT; 1002 } 1003 1004 /** 1005 * nf_ct_resolve_clash - attempt to handle clash without packet drop 1006 * 1007 * @skb: skb that causes the clash 1008 * @h: tuplehash of the clashing entry already in table 1009 * @reply_hash: hash slot for reply direction 1010 * 1011 * A conntrack entry can be inserted to the connection tracking table 1012 * if there is no existing entry with an identical tuple. 1013 * 1014 * If there is one, @skb (and the assocated, unconfirmed conntrack) has 1015 * to be dropped. In case @skb is retransmitted, next conntrack lookup 1016 * will find the already-existing entry. 1017 * 1018 * The major problem with such packet drop is the extra delay added by 1019 * the packet loss -- it will take some time for a retransmit to occur 1020 * (or the sender to time out when waiting for a reply). 1021 * 1022 * This function attempts to handle the situation without packet drop. 1023 * 1024 * If @skb has no NAT transformation or if the colliding entries are 1025 * exactly the same, only the to-be-confirmed conntrack entry is discarded 1026 * and @skb is associated with the conntrack entry already in the table. 1027 * 1028 * Failing that, the new, unconfirmed conntrack is still added to the table 1029 * provided that the collision only occurs in the ORIGINAL direction. 1030 * The new entry will be added after the existing one in the hash list, 1031 * so packets in the ORIGINAL direction will continue to match the existing 1032 * entry. The new entry will also have a fixed timeout so it expires -- 1033 * due to the collision, it will not see bidirectional traffic. 1034 * 1035 * Returns NF_DROP if the clash could not be resolved. 1036 */ 1037 static __cold noinline int 1038 nf_ct_resolve_clash(struct sk_buff *skb, struct nf_conntrack_tuple_hash *h, 1039 u32 reply_hash) 1040 { 1041 /* This is the conntrack entry already in hashes that won race. */ 1042 struct nf_conn *ct = nf_ct_tuplehash_to_ctrack(h); 1043 const struct nf_conntrack_l4proto *l4proto; 1044 enum ip_conntrack_info ctinfo; 1045 struct nf_conn *loser_ct; 1046 struct net *net; 1047 int ret; 1048 1049 loser_ct = nf_ct_get(skb, &ctinfo); 1050 net = nf_ct_net(loser_ct); 1051 1052 l4proto = nf_ct_l4proto_find(nf_ct_protonum(ct)); 1053 if (!l4proto->allow_clash) 1054 goto drop; 1055 1056 ret = __nf_ct_resolve_clash(skb, h); 1057 if (ret == NF_ACCEPT) 1058 return ret; 1059 1060 ret = nf_ct_resolve_clash_harder(skb, reply_hash); 1061 if (ret == NF_ACCEPT) 1062 return ret; 1063 1064 drop: 1065 nf_ct_add_to_dying_list(loser_ct); 1066 NF_CT_STAT_INC(net, drop); 1067 NF_CT_STAT_INC(net, insert_failed); 1068 return NF_DROP; 1069 } 1070 1071 /* Confirm a connection given skb; places it in hash table */ 1072 int 1073 __nf_conntrack_confirm(struct sk_buff *skb) 1074 { 1075 const struct nf_conntrack_zone *zone; 1076 unsigned int hash, reply_hash; 1077 struct nf_conntrack_tuple_hash *h; 1078 struct nf_conn *ct; 1079 struct nf_conn_help *help; 1080 struct hlist_nulls_node *n; 1081 enum ip_conntrack_info ctinfo; 1082 struct net *net; 1083 unsigned int sequence; 1084 int ret = NF_DROP; 1085 1086 ct = nf_ct_get(skb, &ctinfo); 1087 net = nf_ct_net(ct); 1088 1089 /* ipt_REJECT uses nf_conntrack_attach to attach related 1090 ICMP/TCP RST packets in other direction. Actual packet 1091 which created connection will be IP_CT_NEW or for an 1092 expected connection, IP_CT_RELATED. */ 1093 if (CTINFO2DIR(ctinfo) != IP_CT_DIR_ORIGINAL) 1094 return NF_ACCEPT; 1095 1096 zone = nf_ct_zone(ct); 1097 local_bh_disable(); 1098 1099 do { 1100 sequence = read_seqcount_begin(&nf_conntrack_generation); 1101 /* reuse the hash saved before */ 1102 hash = *(unsigned long *)&ct->tuplehash[IP_CT_DIR_REPLY].hnnode.pprev; 1103 hash = scale_hash(hash); 1104 reply_hash = hash_conntrack(net, 1105 &ct->tuplehash[IP_CT_DIR_REPLY].tuple); 1106 1107 } while (nf_conntrack_double_lock(net, hash, reply_hash, sequence)); 1108 1109 /* We're not in hash table, and we refuse to set up related 1110 * connections for unconfirmed conns. But packet copies and 1111 * REJECT will give spurious warnings here. 1112 */ 1113 1114 /* Another skb with the same unconfirmed conntrack may 1115 * win the race. This may happen for bridge(br_flood) 1116 * or broadcast/multicast packets do skb_clone with 1117 * unconfirmed conntrack. 1118 */ 1119 if (unlikely(nf_ct_is_confirmed(ct))) { 1120 WARN_ON_ONCE(1); 1121 nf_conntrack_double_unlock(hash, reply_hash); 1122 local_bh_enable(); 1123 return NF_DROP; 1124 } 1125 1126 pr_debug("Confirming conntrack %p\n", ct); 1127 /* We have to check the DYING flag after unlink to prevent 1128 * a race against nf_ct_get_next_corpse() possibly called from 1129 * user context, else we insert an already 'dead' hash, blocking 1130 * further use of that particular connection -JM. 1131 */ 1132 nf_ct_del_from_dying_or_unconfirmed_list(ct); 1133 1134 if (unlikely(nf_ct_is_dying(ct))) { 1135 nf_ct_add_to_dying_list(ct); 1136 NF_CT_STAT_INC(net, insert_failed); 1137 goto dying; 1138 } 1139 1140 /* See if there's one in the list already, including reverse: 1141 NAT could have grabbed it without realizing, since we're 1142 not in the hash. If there is, we lost race. */ 1143 hlist_nulls_for_each_entry(h, n, &nf_conntrack_hash[hash], hnnode) 1144 if (nf_ct_key_equal(h, &ct->tuplehash[IP_CT_DIR_ORIGINAL].tuple, 1145 zone, net)) 1146 goto out; 1147 1148 hlist_nulls_for_each_entry(h, n, &nf_conntrack_hash[reply_hash], hnnode) 1149 if (nf_ct_key_equal(h, &ct->tuplehash[IP_CT_DIR_REPLY].tuple, 1150 zone, net)) 1151 goto out; 1152 1153 /* Timer relative to confirmation time, not original 1154 setting time, otherwise we'd get timer wrap in 1155 weird delay cases. */ 1156 ct->timeout += nfct_time_stamp; 1157 1158 __nf_conntrack_insert_prepare(ct); 1159 1160 /* Since the lookup is lockless, hash insertion must be done after 1161 * starting the timer and setting the CONFIRMED bit. The RCU barriers 1162 * guarantee that no other CPU can find the conntrack before the above 1163 * stores are visible. 1164 */ 1165 __nf_conntrack_hash_insert(ct, hash, reply_hash); 1166 nf_conntrack_double_unlock(hash, reply_hash); 1167 local_bh_enable(); 1168 1169 help = nfct_help(ct); 1170 if (help && help->helper) 1171 nf_conntrack_event_cache(IPCT_HELPER, ct); 1172 1173 nf_conntrack_event_cache(master_ct(ct) ? 1174 IPCT_RELATED : IPCT_NEW, ct); 1175 return NF_ACCEPT; 1176 1177 out: 1178 ret = nf_ct_resolve_clash(skb, h, reply_hash); 1179 dying: 1180 nf_conntrack_double_unlock(hash, reply_hash); 1181 local_bh_enable(); 1182 return ret; 1183 } 1184 EXPORT_SYMBOL_GPL(__nf_conntrack_confirm); 1185 1186 /* Returns true if a connection correspondings to the tuple (required 1187 for NAT). */ 1188 int 1189 nf_conntrack_tuple_taken(const struct nf_conntrack_tuple *tuple, 1190 const struct nf_conn *ignored_conntrack) 1191 { 1192 struct net *net = nf_ct_net(ignored_conntrack); 1193 const struct nf_conntrack_zone *zone; 1194 struct nf_conntrack_tuple_hash *h; 1195 struct hlist_nulls_head *ct_hash; 1196 unsigned int hash, hsize; 1197 struct hlist_nulls_node *n; 1198 struct nf_conn *ct; 1199 1200 zone = nf_ct_zone(ignored_conntrack); 1201 1202 rcu_read_lock(); 1203 begin: 1204 nf_conntrack_get_ht(&ct_hash, &hsize); 1205 hash = __hash_conntrack(net, tuple, hsize); 1206 1207 hlist_nulls_for_each_entry_rcu(h, n, &ct_hash[hash], hnnode) { 1208 ct = nf_ct_tuplehash_to_ctrack(h); 1209 1210 if (ct == ignored_conntrack) 1211 continue; 1212 1213 if (nf_ct_is_expired(ct)) { 1214 nf_ct_gc_expired(ct); 1215 continue; 1216 } 1217 1218 if (nf_ct_key_equal(h, tuple, zone, net)) { 1219 /* Tuple is taken already, so caller will need to find 1220 * a new source port to use. 1221 * 1222 * Only exception: 1223 * If the *original tuples* are identical, then both 1224 * conntracks refer to the same flow. 1225 * This is a rare situation, it can occur e.g. when 1226 * more than one UDP packet is sent from same socket 1227 * in different threads. 1228 * 1229 * Let nf_ct_resolve_clash() deal with this later. 1230 */ 1231 if (nf_ct_tuple_equal(&ignored_conntrack->tuplehash[IP_CT_DIR_ORIGINAL].tuple, 1232 &ct->tuplehash[IP_CT_DIR_ORIGINAL].tuple)) 1233 continue; 1234 1235 NF_CT_STAT_INC_ATOMIC(net, found); 1236 rcu_read_unlock(); 1237 return 1; 1238 } 1239 } 1240 1241 if (get_nulls_value(n) != hash) { 1242 NF_CT_STAT_INC_ATOMIC(net, search_restart); 1243 goto begin; 1244 } 1245 1246 rcu_read_unlock(); 1247 1248 return 0; 1249 } 1250 EXPORT_SYMBOL_GPL(nf_conntrack_tuple_taken); 1251 1252 #define NF_CT_EVICTION_RANGE 8 1253 1254 /* There's a small race here where we may free a just-assured 1255 connection. Too bad: we're in trouble anyway. */ 1256 static unsigned int early_drop_list(struct net *net, 1257 struct hlist_nulls_head *head) 1258 { 1259 struct nf_conntrack_tuple_hash *h; 1260 struct hlist_nulls_node *n; 1261 unsigned int drops = 0; 1262 struct nf_conn *tmp; 1263 1264 hlist_nulls_for_each_entry_rcu(h, n, head, hnnode) { 1265 tmp = nf_ct_tuplehash_to_ctrack(h); 1266 1267 if (test_bit(IPS_OFFLOAD_BIT, &tmp->status)) 1268 continue; 1269 1270 if (nf_ct_is_expired(tmp)) { 1271 nf_ct_gc_expired(tmp); 1272 continue; 1273 } 1274 1275 if (test_bit(IPS_ASSURED_BIT, &tmp->status) || 1276 !net_eq(nf_ct_net(tmp), net) || 1277 nf_ct_is_dying(tmp)) 1278 continue; 1279 1280 if (!atomic_inc_not_zero(&tmp->ct_general.use)) 1281 continue; 1282 1283 /* kill only if still in same netns -- might have moved due to 1284 * SLAB_TYPESAFE_BY_RCU rules. 1285 * 1286 * We steal the timer reference. If that fails timer has 1287 * already fired or someone else deleted it. Just drop ref 1288 * and move to next entry. 1289 */ 1290 if (net_eq(nf_ct_net(tmp), net) && 1291 nf_ct_is_confirmed(tmp) && 1292 nf_ct_delete(tmp, 0, 0)) 1293 drops++; 1294 1295 nf_ct_put(tmp); 1296 } 1297 1298 return drops; 1299 } 1300 1301 static noinline int early_drop(struct net *net, unsigned int hash) 1302 { 1303 unsigned int i, bucket; 1304 1305 for (i = 0; i < NF_CT_EVICTION_RANGE; i++) { 1306 struct hlist_nulls_head *ct_hash; 1307 unsigned int hsize, drops; 1308 1309 rcu_read_lock(); 1310 nf_conntrack_get_ht(&ct_hash, &hsize); 1311 if (!i) 1312 bucket = reciprocal_scale(hash, hsize); 1313 else 1314 bucket = (bucket + 1) % hsize; 1315 1316 drops = early_drop_list(net, &ct_hash[bucket]); 1317 rcu_read_unlock(); 1318 1319 if (drops) { 1320 NF_CT_STAT_ADD_ATOMIC(net, early_drop, drops); 1321 return true; 1322 } 1323 } 1324 1325 return false; 1326 } 1327 1328 static bool gc_worker_skip_ct(const struct nf_conn *ct) 1329 { 1330 return !nf_ct_is_confirmed(ct) || nf_ct_is_dying(ct); 1331 } 1332 1333 static bool gc_worker_can_early_drop(const struct nf_conn *ct) 1334 { 1335 const struct nf_conntrack_l4proto *l4proto; 1336 1337 if (!test_bit(IPS_ASSURED_BIT, &ct->status)) 1338 return true; 1339 1340 l4proto = nf_ct_l4proto_find(nf_ct_protonum(ct)); 1341 if (l4proto->can_early_drop && l4proto->can_early_drop(ct)) 1342 return true; 1343 1344 return false; 1345 } 1346 1347 static void gc_worker(struct work_struct *work) 1348 { 1349 unsigned int min_interval = max(HZ / GC_MAX_BUCKETS_DIV, 1u); 1350 unsigned int i, goal, buckets = 0, expired_count = 0; 1351 unsigned int nf_conntrack_max95 = 0; 1352 struct conntrack_gc_work *gc_work; 1353 unsigned int ratio, scanned = 0; 1354 unsigned long next_run; 1355 1356 gc_work = container_of(work, struct conntrack_gc_work, dwork.work); 1357 1358 goal = nf_conntrack_htable_size / GC_MAX_BUCKETS_DIV; 1359 i = gc_work->last_bucket; 1360 if (gc_work->early_drop) 1361 nf_conntrack_max95 = nf_conntrack_max / 100u * 95u; 1362 1363 do { 1364 struct nf_conntrack_tuple_hash *h; 1365 struct hlist_nulls_head *ct_hash; 1366 struct hlist_nulls_node *n; 1367 unsigned int hashsz; 1368 struct nf_conn *tmp; 1369 1370 i++; 1371 rcu_read_lock(); 1372 1373 nf_conntrack_get_ht(&ct_hash, &hashsz); 1374 if (i >= hashsz) 1375 i = 0; 1376 1377 hlist_nulls_for_each_entry_rcu(h, n, &ct_hash[i], hnnode) { 1378 struct net *net; 1379 1380 tmp = nf_ct_tuplehash_to_ctrack(h); 1381 1382 scanned++; 1383 if (test_bit(IPS_OFFLOAD_BIT, &tmp->status)) { 1384 nf_ct_offload_timeout(tmp); 1385 continue; 1386 } 1387 1388 if (nf_ct_is_expired(tmp)) { 1389 nf_ct_gc_expired(tmp); 1390 expired_count++; 1391 continue; 1392 } 1393 1394 if (nf_conntrack_max95 == 0 || gc_worker_skip_ct(tmp)) 1395 continue; 1396 1397 net = nf_ct_net(tmp); 1398 if (atomic_read(&net->ct.count) < nf_conntrack_max95) 1399 continue; 1400 1401 /* need to take reference to avoid possible races */ 1402 if (!atomic_inc_not_zero(&tmp->ct_general.use)) 1403 continue; 1404 1405 if (gc_worker_skip_ct(tmp)) { 1406 nf_ct_put(tmp); 1407 continue; 1408 } 1409 1410 if (gc_worker_can_early_drop(tmp)) 1411 nf_ct_kill(tmp); 1412 1413 nf_ct_put(tmp); 1414 } 1415 1416 /* could check get_nulls_value() here and restart if ct 1417 * was moved to another chain. But given gc is best-effort 1418 * we will just continue with next hash slot. 1419 */ 1420 rcu_read_unlock(); 1421 cond_resched(); 1422 } while (++buckets < goal); 1423 1424 if (gc_work->exiting) 1425 return; 1426 1427 /* 1428 * Eviction will normally happen from the packet path, and not 1429 * from this gc worker. 1430 * 1431 * This worker is only here to reap expired entries when system went 1432 * idle after a busy period. 1433 * 1434 * The heuristics below are supposed to balance conflicting goals: 1435 * 1436 * 1. Minimize time until we notice a stale entry 1437 * 2. Maximize scan intervals to not waste cycles 1438 * 1439 * Normally, expire ratio will be close to 0. 1440 * 1441 * As soon as a sizeable fraction of the entries have expired 1442 * increase scan frequency. 1443 */ 1444 ratio = scanned ? expired_count * 100 / scanned : 0; 1445 if (ratio > GC_EVICT_RATIO) { 1446 gc_work->next_gc_run = min_interval; 1447 } else { 1448 unsigned int max = GC_MAX_SCAN_JIFFIES / GC_MAX_BUCKETS_DIV; 1449 1450 BUILD_BUG_ON((GC_MAX_SCAN_JIFFIES / GC_MAX_BUCKETS_DIV) == 0); 1451 1452 gc_work->next_gc_run += min_interval; 1453 if (gc_work->next_gc_run > max) 1454 gc_work->next_gc_run = max; 1455 } 1456 1457 next_run = gc_work->next_gc_run; 1458 gc_work->last_bucket = i; 1459 gc_work->early_drop = false; 1460 queue_delayed_work(system_power_efficient_wq, &gc_work->dwork, next_run); 1461 } 1462 1463 static void conntrack_gc_work_init(struct conntrack_gc_work *gc_work) 1464 { 1465 INIT_DEFERRABLE_WORK(&gc_work->dwork, gc_worker); 1466 gc_work->next_gc_run = HZ; 1467 gc_work->exiting = false; 1468 } 1469 1470 static struct nf_conn * 1471 __nf_conntrack_alloc(struct net *net, 1472 const struct nf_conntrack_zone *zone, 1473 const struct nf_conntrack_tuple *orig, 1474 const struct nf_conntrack_tuple *repl, 1475 gfp_t gfp, u32 hash) 1476 { 1477 struct nf_conn *ct; 1478 1479 /* We don't want any race condition at early drop stage */ 1480 atomic_inc(&net->ct.count); 1481 1482 if (nf_conntrack_max && 1483 unlikely(atomic_read(&net->ct.count) > nf_conntrack_max)) { 1484 if (!early_drop(net, hash)) { 1485 if (!conntrack_gc_work.early_drop) 1486 conntrack_gc_work.early_drop = true; 1487 atomic_dec(&net->ct.count); 1488 net_warn_ratelimited("nf_conntrack: table full, dropping packet\n"); 1489 return ERR_PTR(-ENOMEM); 1490 } 1491 } 1492 1493 /* 1494 * Do not use kmem_cache_zalloc(), as this cache uses 1495 * SLAB_TYPESAFE_BY_RCU. 1496 */ 1497 ct = kmem_cache_alloc(nf_conntrack_cachep, gfp); 1498 if (ct == NULL) 1499 goto out; 1500 1501 spin_lock_init(&ct->lock); 1502 ct->tuplehash[IP_CT_DIR_ORIGINAL].tuple = *orig; 1503 ct->tuplehash[IP_CT_DIR_ORIGINAL].hnnode.pprev = NULL; 1504 ct->tuplehash[IP_CT_DIR_REPLY].tuple = *repl; 1505 /* save hash for reusing when confirming */ 1506 *(unsigned long *)(&ct->tuplehash[IP_CT_DIR_REPLY].hnnode.pprev) = hash; 1507 ct->status = 0; 1508 ct->timeout = 0; 1509 write_pnet(&ct->ct_net, net); 1510 memset(&ct->__nfct_init_offset, 0, 1511 offsetof(struct nf_conn, proto) - 1512 offsetof(struct nf_conn, __nfct_init_offset)); 1513 1514 nf_ct_zone_add(ct, zone); 1515 1516 /* Because we use RCU lookups, we set ct_general.use to zero before 1517 * this is inserted in any list. 1518 */ 1519 atomic_set(&ct->ct_general.use, 0); 1520 return ct; 1521 out: 1522 atomic_dec(&net->ct.count); 1523 return ERR_PTR(-ENOMEM); 1524 } 1525 1526 struct nf_conn *nf_conntrack_alloc(struct net *net, 1527 const struct nf_conntrack_zone *zone, 1528 const struct nf_conntrack_tuple *orig, 1529 const struct nf_conntrack_tuple *repl, 1530 gfp_t gfp) 1531 { 1532 return __nf_conntrack_alloc(net, zone, orig, repl, gfp, 0); 1533 } 1534 EXPORT_SYMBOL_GPL(nf_conntrack_alloc); 1535 1536 void nf_conntrack_free(struct nf_conn *ct) 1537 { 1538 struct net *net = nf_ct_net(ct); 1539 1540 /* A freed object has refcnt == 0, that's 1541 * the golden rule for SLAB_TYPESAFE_BY_RCU 1542 */ 1543 WARN_ON(atomic_read(&ct->ct_general.use) != 0); 1544 1545 nf_ct_ext_destroy(ct); 1546 kmem_cache_free(nf_conntrack_cachep, ct); 1547 smp_mb__before_atomic(); 1548 atomic_dec(&net->ct.count); 1549 } 1550 EXPORT_SYMBOL_GPL(nf_conntrack_free); 1551 1552 1553 /* Allocate a new conntrack: we return -ENOMEM if classification 1554 failed due to stress. Otherwise it really is unclassifiable. */ 1555 static noinline struct nf_conntrack_tuple_hash * 1556 init_conntrack(struct net *net, struct nf_conn *tmpl, 1557 const struct nf_conntrack_tuple *tuple, 1558 struct sk_buff *skb, 1559 unsigned int dataoff, u32 hash) 1560 { 1561 struct nf_conn *ct; 1562 struct nf_conn_help *help; 1563 struct nf_conntrack_tuple repl_tuple; 1564 struct nf_conntrack_ecache *ecache; 1565 struct nf_conntrack_expect *exp = NULL; 1566 const struct nf_conntrack_zone *zone; 1567 struct nf_conn_timeout *timeout_ext; 1568 struct nf_conntrack_zone tmp; 1569 1570 if (!nf_ct_invert_tuple(&repl_tuple, tuple)) { 1571 pr_debug("Can't invert tuple.\n"); 1572 return NULL; 1573 } 1574 1575 zone = nf_ct_zone_tmpl(tmpl, skb, &tmp); 1576 ct = __nf_conntrack_alloc(net, zone, tuple, &repl_tuple, GFP_ATOMIC, 1577 hash); 1578 if (IS_ERR(ct)) 1579 return (struct nf_conntrack_tuple_hash *)ct; 1580 1581 if (!nf_ct_add_synproxy(ct, tmpl)) { 1582 nf_conntrack_free(ct); 1583 return ERR_PTR(-ENOMEM); 1584 } 1585 1586 timeout_ext = tmpl ? nf_ct_timeout_find(tmpl) : NULL; 1587 1588 if (timeout_ext) 1589 nf_ct_timeout_ext_add(ct, rcu_dereference(timeout_ext->timeout), 1590 GFP_ATOMIC); 1591 1592 nf_ct_acct_ext_add(ct, GFP_ATOMIC); 1593 nf_ct_tstamp_ext_add(ct, GFP_ATOMIC); 1594 nf_ct_labels_ext_add(ct); 1595 1596 ecache = tmpl ? nf_ct_ecache_find(tmpl) : NULL; 1597 nf_ct_ecache_ext_add(ct, ecache ? ecache->ctmask : 0, 1598 ecache ? ecache->expmask : 0, 1599 GFP_ATOMIC); 1600 1601 local_bh_disable(); 1602 if (net->ct.expect_count) { 1603 spin_lock(&nf_conntrack_expect_lock); 1604 exp = nf_ct_find_expectation(net, zone, tuple); 1605 if (exp) { 1606 pr_debug("expectation arrives ct=%p exp=%p\n", 1607 ct, exp); 1608 /* Welcome, Mr. Bond. We've been expecting you... */ 1609 __set_bit(IPS_EXPECTED_BIT, &ct->status); 1610 /* exp->master safe, refcnt bumped in nf_ct_find_expectation */ 1611 ct->master = exp->master; 1612 if (exp->helper) { 1613 help = nf_ct_helper_ext_add(ct, GFP_ATOMIC); 1614 if (help) 1615 rcu_assign_pointer(help->helper, exp->helper); 1616 } 1617 1618 #ifdef CONFIG_NF_CONNTRACK_MARK 1619 ct->mark = exp->master->mark; 1620 #endif 1621 #ifdef CONFIG_NF_CONNTRACK_SECMARK 1622 ct->secmark = exp->master->secmark; 1623 #endif 1624 NF_CT_STAT_INC(net, expect_new); 1625 } 1626 spin_unlock(&nf_conntrack_expect_lock); 1627 } 1628 if (!exp) 1629 __nf_ct_try_assign_helper(ct, tmpl, GFP_ATOMIC); 1630 1631 /* Now it is inserted into the unconfirmed list, bump refcount */ 1632 nf_conntrack_get(&ct->ct_general); 1633 nf_ct_add_to_unconfirmed_list(ct); 1634 1635 local_bh_enable(); 1636 1637 if (exp) { 1638 if (exp->expectfn) 1639 exp->expectfn(ct, exp); 1640 nf_ct_expect_put(exp); 1641 } 1642 1643 return &ct->tuplehash[IP_CT_DIR_ORIGINAL]; 1644 } 1645 1646 /* On success, returns 0, sets skb->_nfct | ctinfo */ 1647 static int 1648 resolve_normal_ct(struct nf_conn *tmpl, 1649 struct sk_buff *skb, 1650 unsigned int dataoff, 1651 u_int8_t protonum, 1652 const struct nf_hook_state *state) 1653 { 1654 const struct nf_conntrack_zone *zone; 1655 struct nf_conntrack_tuple tuple; 1656 struct nf_conntrack_tuple_hash *h; 1657 enum ip_conntrack_info ctinfo; 1658 struct nf_conntrack_zone tmp; 1659 struct nf_conn *ct; 1660 u32 hash; 1661 1662 if (!nf_ct_get_tuple(skb, skb_network_offset(skb), 1663 dataoff, state->pf, protonum, state->net, 1664 &tuple)) { 1665 pr_debug("Can't get tuple\n"); 1666 return 0; 1667 } 1668 1669 /* look for tuple match */ 1670 zone = nf_ct_zone_tmpl(tmpl, skb, &tmp); 1671 hash = hash_conntrack_raw(&tuple, state->net); 1672 h = __nf_conntrack_find_get(state->net, zone, &tuple, hash); 1673 if (!h) { 1674 h = init_conntrack(state->net, tmpl, &tuple, 1675 skb, dataoff, hash); 1676 if (!h) 1677 return 0; 1678 if (IS_ERR(h)) 1679 return PTR_ERR(h); 1680 } 1681 ct = nf_ct_tuplehash_to_ctrack(h); 1682 1683 /* It exists; we have (non-exclusive) reference. */ 1684 if (NF_CT_DIRECTION(h) == IP_CT_DIR_REPLY) { 1685 ctinfo = IP_CT_ESTABLISHED_REPLY; 1686 } else { 1687 /* Once we've had two way comms, always ESTABLISHED. */ 1688 if (test_bit(IPS_SEEN_REPLY_BIT, &ct->status)) { 1689 pr_debug("normal packet for %p\n", ct); 1690 ctinfo = IP_CT_ESTABLISHED; 1691 } else if (test_bit(IPS_EXPECTED_BIT, &ct->status)) { 1692 pr_debug("related packet for %p\n", ct); 1693 ctinfo = IP_CT_RELATED; 1694 } else { 1695 pr_debug("new packet for %p\n", ct); 1696 ctinfo = IP_CT_NEW; 1697 } 1698 } 1699 nf_ct_set(skb, ct, ctinfo); 1700 return 0; 1701 } 1702 1703 /* 1704 * icmp packets need special treatment to handle error messages that are 1705 * related to a connection. 1706 * 1707 * Callers need to check if skb has a conntrack assigned when this 1708 * helper returns; in such case skb belongs to an already known connection. 1709 */ 1710 static unsigned int __cold 1711 nf_conntrack_handle_icmp(struct nf_conn *tmpl, 1712 struct sk_buff *skb, 1713 unsigned int dataoff, 1714 u8 protonum, 1715 const struct nf_hook_state *state) 1716 { 1717 int ret; 1718 1719 if (state->pf == NFPROTO_IPV4 && protonum == IPPROTO_ICMP) 1720 ret = nf_conntrack_icmpv4_error(tmpl, skb, dataoff, state); 1721 #if IS_ENABLED(CONFIG_IPV6) 1722 else if (state->pf == NFPROTO_IPV6 && protonum == IPPROTO_ICMPV6) 1723 ret = nf_conntrack_icmpv6_error(tmpl, skb, dataoff, state); 1724 #endif 1725 else 1726 return NF_ACCEPT; 1727 1728 if (ret <= 0) { 1729 NF_CT_STAT_INC_ATOMIC(state->net, error); 1730 NF_CT_STAT_INC_ATOMIC(state->net, invalid); 1731 } 1732 1733 return ret; 1734 } 1735 1736 static int generic_packet(struct nf_conn *ct, struct sk_buff *skb, 1737 enum ip_conntrack_info ctinfo) 1738 { 1739 const unsigned int *timeout = nf_ct_timeout_lookup(ct); 1740 1741 if (!timeout) 1742 timeout = &nf_generic_pernet(nf_ct_net(ct))->timeout; 1743 1744 nf_ct_refresh_acct(ct, ctinfo, skb, *timeout); 1745 return NF_ACCEPT; 1746 } 1747 1748 /* Returns verdict for packet, or -1 for invalid. */ 1749 static int nf_conntrack_handle_packet(struct nf_conn *ct, 1750 struct sk_buff *skb, 1751 unsigned int dataoff, 1752 enum ip_conntrack_info ctinfo, 1753 const struct nf_hook_state *state) 1754 { 1755 switch (nf_ct_protonum(ct)) { 1756 case IPPROTO_TCP: 1757 return nf_conntrack_tcp_packet(ct, skb, dataoff, 1758 ctinfo, state); 1759 case IPPROTO_UDP: 1760 return nf_conntrack_udp_packet(ct, skb, dataoff, 1761 ctinfo, state); 1762 case IPPROTO_ICMP: 1763 return nf_conntrack_icmp_packet(ct, skb, ctinfo, state); 1764 #if IS_ENABLED(CONFIG_IPV6) 1765 case IPPROTO_ICMPV6: 1766 return nf_conntrack_icmpv6_packet(ct, skb, ctinfo, state); 1767 #endif 1768 #ifdef CONFIG_NF_CT_PROTO_UDPLITE 1769 case IPPROTO_UDPLITE: 1770 return nf_conntrack_udplite_packet(ct, skb, dataoff, 1771 ctinfo, state); 1772 #endif 1773 #ifdef CONFIG_NF_CT_PROTO_SCTP 1774 case IPPROTO_SCTP: 1775 return nf_conntrack_sctp_packet(ct, skb, dataoff, 1776 ctinfo, state); 1777 #endif 1778 #ifdef CONFIG_NF_CT_PROTO_DCCP 1779 case IPPROTO_DCCP: 1780 return nf_conntrack_dccp_packet(ct, skb, dataoff, 1781 ctinfo, state); 1782 #endif 1783 #ifdef CONFIG_NF_CT_PROTO_GRE 1784 case IPPROTO_GRE: 1785 return nf_conntrack_gre_packet(ct, skb, dataoff, 1786 ctinfo, state); 1787 #endif 1788 } 1789 1790 return generic_packet(ct, skb, ctinfo); 1791 } 1792 1793 unsigned int 1794 nf_conntrack_in(struct sk_buff *skb, const struct nf_hook_state *state) 1795 { 1796 enum ip_conntrack_info ctinfo; 1797 struct nf_conn *ct, *tmpl; 1798 u_int8_t protonum; 1799 int dataoff, ret; 1800 1801 tmpl = nf_ct_get(skb, &ctinfo); 1802 if (tmpl || ctinfo == IP_CT_UNTRACKED) { 1803 /* Previously seen (loopback or untracked)? Ignore. */ 1804 if ((tmpl && !nf_ct_is_template(tmpl)) || 1805 ctinfo == IP_CT_UNTRACKED) { 1806 NF_CT_STAT_INC_ATOMIC(state->net, ignore); 1807 return NF_ACCEPT; 1808 } 1809 skb->_nfct = 0; 1810 } 1811 1812 /* rcu_read_lock()ed by nf_hook_thresh */ 1813 dataoff = get_l4proto(skb, skb_network_offset(skb), state->pf, &protonum); 1814 if (dataoff <= 0) { 1815 pr_debug("not prepared to track yet or error occurred\n"); 1816 NF_CT_STAT_INC_ATOMIC(state->net, error); 1817 NF_CT_STAT_INC_ATOMIC(state->net, invalid); 1818 ret = NF_ACCEPT; 1819 goto out; 1820 } 1821 1822 if (protonum == IPPROTO_ICMP || protonum == IPPROTO_ICMPV6) { 1823 ret = nf_conntrack_handle_icmp(tmpl, skb, dataoff, 1824 protonum, state); 1825 if (ret <= 0) { 1826 ret = -ret; 1827 goto out; 1828 } 1829 /* ICMP[v6] protocol trackers may assign one conntrack. */ 1830 if (skb->_nfct) 1831 goto out; 1832 } 1833 repeat: 1834 ret = resolve_normal_ct(tmpl, skb, dataoff, 1835 protonum, state); 1836 if (ret < 0) { 1837 /* Too stressed to deal. */ 1838 NF_CT_STAT_INC_ATOMIC(state->net, drop); 1839 ret = NF_DROP; 1840 goto out; 1841 } 1842 1843 ct = nf_ct_get(skb, &ctinfo); 1844 if (!ct) { 1845 /* Not valid part of a connection */ 1846 NF_CT_STAT_INC_ATOMIC(state->net, invalid); 1847 ret = NF_ACCEPT; 1848 goto out; 1849 } 1850 1851 ret = nf_conntrack_handle_packet(ct, skb, dataoff, ctinfo, state); 1852 if (ret <= 0) { 1853 /* Invalid: inverse of the return code tells 1854 * the netfilter core what to do */ 1855 pr_debug("nf_conntrack_in: Can't track with proto module\n"); 1856 nf_conntrack_put(&ct->ct_general); 1857 skb->_nfct = 0; 1858 NF_CT_STAT_INC_ATOMIC(state->net, invalid); 1859 if (ret == -NF_DROP) 1860 NF_CT_STAT_INC_ATOMIC(state->net, drop); 1861 /* Special case: TCP tracker reports an attempt to reopen a 1862 * closed/aborted connection. We have to go back and create a 1863 * fresh conntrack. 1864 */ 1865 if (ret == -NF_REPEAT) 1866 goto repeat; 1867 ret = -ret; 1868 goto out; 1869 } 1870 1871 if (ctinfo == IP_CT_ESTABLISHED_REPLY && 1872 !test_and_set_bit(IPS_SEEN_REPLY_BIT, &ct->status)) 1873 nf_conntrack_event_cache(IPCT_REPLY, ct); 1874 out: 1875 if (tmpl) 1876 nf_ct_put(tmpl); 1877 1878 return ret; 1879 } 1880 EXPORT_SYMBOL_GPL(nf_conntrack_in); 1881 1882 /* Alter reply tuple (maybe alter helper). This is for NAT, and is 1883 implicitly racy: see __nf_conntrack_confirm */ 1884 void nf_conntrack_alter_reply(struct nf_conn *ct, 1885 const struct nf_conntrack_tuple *newreply) 1886 { 1887 struct nf_conn_help *help = nfct_help(ct); 1888 1889 /* Should be unconfirmed, so not in hash table yet */ 1890 WARN_ON(nf_ct_is_confirmed(ct)); 1891 1892 pr_debug("Altering reply tuple of %p to ", ct); 1893 nf_ct_dump_tuple(newreply); 1894 1895 ct->tuplehash[IP_CT_DIR_REPLY].tuple = *newreply; 1896 if (ct->master || (help && !hlist_empty(&help->expectations))) 1897 return; 1898 1899 rcu_read_lock(); 1900 __nf_ct_try_assign_helper(ct, NULL, GFP_ATOMIC); 1901 rcu_read_unlock(); 1902 } 1903 EXPORT_SYMBOL_GPL(nf_conntrack_alter_reply); 1904 1905 /* Refresh conntrack for this many jiffies and do accounting if do_acct is 1 */ 1906 void __nf_ct_refresh_acct(struct nf_conn *ct, 1907 enum ip_conntrack_info ctinfo, 1908 const struct sk_buff *skb, 1909 u32 extra_jiffies, 1910 bool do_acct) 1911 { 1912 /* Only update if this is not a fixed timeout */ 1913 if (test_bit(IPS_FIXED_TIMEOUT_BIT, &ct->status)) 1914 goto acct; 1915 1916 /* If not in hash table, timer will not be active yet */ 1917 if (nf_ct_is_confirmed(ct)) 1918 extra_jiffies += nfct_time_stamp; 1919 1920 if (READ_ONCE(ct->timeout) != extra_jiffies) 1921 WRITE_ONCE(ct->timeout, extra_jiffies); 1922 acct: 1923 if (do_acct) 1924 nf_ct_acct_update(ct, CTINFO2DIR(ctinfo), skb->len); 1925 } 1926 EXPORT_SYMBOL_GPL(__nf_ct_refresh_acct); 1927 1928 bool nf_ct_kill_acct(struct nf_conn *ct, 1929 enum ip_conntrack_info ctinfo, 1930 const struct sk_buff *skb) 1931 { 1932 nf_ct_acct_update(ct, CTINFO2DIR(ctinfo), skb->len); 1933 1934 return nf_ct_delete(ct, 0, 0); 1935 } 1936 EXPORT_SYMBOL_GPL(nf_ct_kill_acct); 1937 1938 #if IS_ENABLED(CONFIG_NF_CT_NETLINK) 1939 1940 #include <linux/netfilter/nfnetlink.h> 1941 #include <linux/netfilter/nfnetlink_conntrack.h> 1942 #include <linux/mutex.h> 1943 1944 /* Generic function for tcp/udp/sctp/dccp and alike. */ 1945 int nf_ct_port_tuple_to_nlattr(struct sk_buff *skb, 1946 const struct nf_conntrack_tuple *tuple) 1947 { 1948 if (nla_put_be16(skb, CTA_PROTO_SRC_PORT, tuple->src.u.tcp.port) || 1949 nla_put_be16(skb, CTA_PROTO_DST_PORT, tuple->dst.u.tcp.port)) 1950 goto nla_put_failure; 1951 return 0; 1952 1953 nla_put_failure: 1954 return -1; 1955 } 1956 EXPORT_SYMBOL_GPL(nf_ct_port_tuple_to_nlattr); 1957 1958 const struct nla_policy nf_ct_port_nla_policy[CTA_PROTO_MAX+1] = { 1959 [CTA_PROTO_SRC_PORT] = { .type = NLA_U16 }, 1960 [CTA_PROTO_DST_PORT] = { .type = NLA_U16 }, 1961 }; 1962 EXPORT_SYMBOL_GPL(nf_ct_port_nla_policy); 1963 1964 int nf_ct_port_nlattr_to_tuple(struct nlattr *tb[], 1965 struct nf_conntrack_tuple *t, 1966 u_int32_t flags) 1967 { 1968 if (flags & CTA_FILTER_FLAG(CTA_PROTO_SRC_PORT)) { 1969 if (!tb[CTA_PROTO_SRC_PORT]) 1970 return -EINVAL; 1971 1972 t->src.u.tcp.port = nla_get_be16(tb[CTA_PROTO_SRC_PORT]); 1973 } 1974 1975 if (flags & CTA_FILTER_FLAG(CTA_PROTO_DST_PORT)) { 1976 if (!tb[CTA_PROTO_DST_PORT]) 1977 return -EINVAL; 1978 1979 t->dst.u.tcp.port = nla_get_be16(tb[CTA_PROTO_DST_PORT]); 1980 } 1981 1982 return 0; 1983 } 1984 EXPORT_SYMBOL_GPL(nf_ct_port_nlattr_to_tuple); 1985 1986 unsigned int nf_ct_port_nlattr_tuple_size(void) 1987 { 1988 static unsigned int size __read_mostly; 1989 1990 if (!size) 1991 size = nla_policy_len(nf_ct_port_nla_policy, CTA_PROTO_MAX + 1); 1992 1993 return size; 1994 } 1995 EXPORT_SYMBOL_GPL(nf_ct_port_nlattr_tuple_size); 1996 #endif 1997 1998 /* Used by ipt_REJECT and ip6t_REJECT. */ 1999 static void nf_conntrack_attach(struct sk_buff *nskb, const struct sk_buff *skb) 2000 { 2001 struct nf_conn *ct; 2002 enum ip_conntrack_info ctinfo; 2003 2004 /* This ICMP is in reverse direction to the packet which caused it */ 2005 ct = nf_ct_get(skb, &ctinfo); 2006 if (CTINFO2DIR(ctinfo) == IP_CT_DIR_ORIGINAL) 2007 ctinfo = IP_CT_RELATED_REPLY; 2008 else 2009 ctinfo = IP_CT_RELATED; 2010 2011 /* Attach to new skbuff, and increment count */ 2012 nf_ct_set(nskb, ct, ctinfo); 2013 nf_conntrack_get(skb_nfct(nskb)); 2014 } 2015 2016 static int __nf_conntrack_update(struct net *net, struct sk_buff *skb, 2017 struct nf_conn *ct, 2018 enum ip_conntrack_info ctinfo) 2019 { 2020 struct nf_conntrack_tuple_hash *h; 2021 struct nf_conntrack_tuple tuple; 2022 struct nf_nat_hook *nat_hook; 2023 unsigned int status; 2024 int dataoff; 2025 u16 l3num; 2026 u8 l4num; 2027 2028 l3num = nf_ct_l3num(ct); 2029 2030 dataoff = get_l4proto(skb, skb_network_offset(skb), l3num, &l4num); 2031 if (dataoff <= 0) 2032 return -1; 2033 2034 if (!nf_ct_get_tuple(skb, skb_network_offset(skb), dataoff, l3num, 2035 l4num, net, &tuple)) 2036 return -1; 2037 2038 if (ct->status & IPS_SRC_NAT) { 2039 memcpy(tuple.src.u3.all, 2040 ct->tuplehash[IP_CT_DIR_ORIGINAL].tuple.src.u3.all, 2041 sizeof(tuple.src.u3.all)); 2042 tuple.src.u.all = 2043 ct->tuplehash[IP_CT_DIR_ORIGINAL].tuple.src.u.all; 2044 } 2045 2046 if (ct->status & IPS_DST_NAT) { 2047 memcpy(tuple.dst.u3.all, 2048 ct->tuplehash[IP_CT_DIR_ORIGINAL].tuple.dst.u3.all, 2049 sizeof(tuple.dst.u3.all)); 2050 tuple.dst.u.all = 2051 ct->tuplehash[IP_CT_DIR_ORIGINAL].tuple.dst.u.all; 2052 } 2053 2054 h = nf_conntrack_find_get(net, nf_ct_zone(ct), &tuple); 2055 if (!h) 2056 return 0; 2057 2058 /* Store status bits of the conntrack that is clashing to re-do NAT 2059 * mangling according to what it has been done already to this packet. 2060 */ 2061 status = ct->status; 2062 2063 nf_ct_put(ct); 2064 ct = nf_ct_tuplehash_to_ctrack(h); 2065 nf_ct_set(skb, ct, ctinfo); 2066 2067 nat_hook = rcu_dereference(nf_nat_hook); 2068 if (!nat_hook) 2069 return 0; 2070 2071 if (status & IPS_SRC_NAT && 2072 nat_hook->manip_pkt(skb, ct, NF_NAT_MANIP_SRC, 2073 IP_CT_DIR_ORIGINAL) == NF_DROP) 2074 return -1; 2075 2076 if (status & IPS_DST_NAT && 2077 nat_hook->manip_pkt(skb, ct, NF_NAT_MANIP_DST, 2078 IP_CT_DIR_ORIGINAL) == NF_DROP) 2079 return -1; 2080 2081 return 0; 2082 } 2083 2084 /* This packet is coming from userspace via nf_queue, complete the packet 2085 * processing after the helper invocation in nf_confirm(). 2086 */ 2087 static int nf_confirm_cthelper(struct sk_buff *skb, struct nf_conn *ct, 2088 enum ip_conntrack_info ctinfo) 2089 { 2090 const struct nf_conntrack_helper *helper; 2091 const struct nf_conn_help *help; 2092 int protoff; 2093 2094 help = nfct_help(ct); 2095 if (!help) 2096 return 0; 2097 2098 helper = rcu_dereference(help->helper); 2099 if (!(helper->flags & NF_CT_HELPER_F_USERSPACE)) 2100 return 0; 2101 2102 switch (nf_ct_l3num(ct)) { 2103 case NFPROTO_IPV4: 2104 protoff = skb_network_offset(skb) + ip_hdrlen(skb); 2105 break; 2106 #if IS_ENABLED(CONFIG_IPV6) 2107 case NFPROTO_IPV6: { 2108 __be16 frag_off; 2109 u8 pnum; 2110 2111 pnum = ipv6_hdr(skb)->nexthdr; 2112 protoff = ipv6_skip_exthdr(skb, sizeof(struct ipv6hdr), &pnum, 2113 &frag_off); 2114 if (protoff < 0 || (frag_off & htons(~0x7)) != 0) 2115 return 0; 2116 break; 2117 } 2118 #endif 2119 default: 2120 return 0; 2121 } 2122 2123 if (test_bit(IPS_SEQ_ADJUST_BIT, &ct->status) && 2124 !nf_is_loopback_packet(skb)) { 2125 if (!nf_ct_seq_adjust(skb, ct, ctinfo, protoff)) { 2126 NF_CT_STAT_INC_ATOMIC(nf_ct_net(ct), drop); 2127 return -1; 2128 } 2129 } 2130 2131 /* We've seen it coming out the other side: confirm it */ 2132 return nf_conntrack_confirm(skb) == NF_DROP ? - 1 : 0; 2133 } 2134 2135 static int nf_conntrack_update(struct net *net, struct sk_buff *skb) 2136 { 2137 enum ip_conntrack_info ctinfo; 2138 struct nf_conn *ct; 2139 int err; 2140 2141 ct = nf_ct_get(skb, &ctinfo); 2142 if (!ct) 2143 return 0; 2144 2145 if (!nf_ct_is_confirmed(ct)) { 2146 err = __nf_conntrack_update(net, skb, ct, ctinfo); 2147 if (err < 0) 2148 return err; 2149 2150 ct = nf_ct_get(skb, &ctinfo); 2151 } 2152 2153 return nf_confirm_cthelper(skb, ct, ctinfo); 2154 } 2155 2156 static bool nf_conntrack_get_tuple_skb(struct nf_conntrack_tuple *dst_tuple, 2157 const struct sk_buff *skb) 2158 { 2159 const struct nf_conntrack_tuple *src_tuple; 2160 const struct nf_conntrack_tuple_hash *hash; 2161 struct nf_conntrack_tuple srctuple; 2162 enum ip_conntrack_info ctinfo; 2163 struct nf_conn *ct; 2164 2165 ct = nf_ct_get(skb, &ctinfo); 2166 if (ct) { 2167 src_tuple = nf_ct_tuple(ct, CTINFO2DIR(ctinfo)); 2168 memcpy(dst_tuple, src_tuple, sizeof(*dst_tuple)); 2169 return true; 2170 } 2171 2172 if (!nf_ct_get_tuplepr(skb, skb_network_offset(skb), 2173 NFPROTO_IPV4, dev_net(skb->dev), 2174 &srctuple)) 2175 return false; 2176 2177 hash = nf_conntrack_find_get(dev_net(skb->dev), 2178 &nf_ct_zone_dflt, 2179 &srctuple); 2180 if (!hash) 2181 return false; 2182 2183 ct = nf_ct_tuplehash_to_ctrack(hash); 2184 src_tuple = nf_ct_tuple(ct, !hash->tuple.dst.dir); 2185 memcpy(dst_tuple, src_tuple, sizeof(*dst_tuple)); 2186 nf_ct_put(ct); 2187 2188 return true; 2189 } 2190 2191 /* Bring out ya dead! */ 2192 static struct nf_conn * 2193 get_next_corpse(int (*iter)(struct nf_conn *i, void *data), 2194 void *data, unsigned int *bucket) 2195 { 2196 struct nf_conntrack_tuple_hash *h; 2197 struct nf_conn *ct; 2198 struct hlist_nulls_node *n; 2199 spinlock_t *lockp; 2200 2201 for (; *bucket < nf_conntrack_htable_size; (*bucket)++) { 2202 lockp = &nf_conntrack_locks[*bucket % CONNTRACK_LOCKS]; 2203 local_bh_disable(); 2204 nf_conntrack_lock(lockp); 2205 if (*bucket < nf_conntrack_htable_size) { 2206 hlist_nulls_for_each_entry(h, n, &nf_conntrack_hash[*bucket], hnnode) { 2207 if (NF_CT_DIRECTION(h) != IP_CT_DIR_REPLY) 2208 continue; 2209 /* All nf_conn objects are added to hash table twice, one 2210 * for original direction tuple, once for the reply tuple. 2211 * 2212 * Exception: In the IPS_NAT_CLASH case, only the reply 2213 * tuple is added (the original tuple already existed for 2214 * a different object). 2215 * 2216 * We only need to call the iterator once for each 2217 * conntrack, so we just use the 'reply' direction 2218 * tuple while iterating. 2219 */ 2220 ct = nf_ct_tuplehash_to_ctrack(h); 2221 if (iter(ct, data)) 2222 goto found; 2223 } 2224 } 2225 spin_unlock(lockp); 2226 local_bh_enable(); 2227 cond_resched(); 2228 } 2229 2230 return NULL; 2231 found: 2232 atomic_inc(&ct->ct_general.use); 2233 spin_unlock(lockp); 2234 local_bh_enable(); 2235 return ct; 2236 } 2237 2238 static void nf_ct_iterate_cleanup(int (*iter)(struct nf_conn *i, void *data), 2239 void *data, u32 portid, int report) 2240 { 2241 unsigned int bucket = 0, sequence; 2242 struct nf_conn *ct; 2243 2244 might_sleep(); 2245 2246 for (;;) { 2247 sequence = read_seqcount_begin(&nf_conntrack_generation); 2248 2249 while ((ct = get_next_corpse(iter, data, &bucket)) != NULL) { 2250 /* Time to push up daises... */ 2251 2252 nf_ct_delete(ct, portid, report); 2253 nf_ct_put(ct); 2254 cond_resched(); 2255 } 2256 2257 if (!read_seqcount_retry(&nf_conntrack_generation, sequence)) 2258 break; 2259 bucket = 0; 2260 } 2261 } 2262 2263 struct iter_data { 2264 int (*iter)(struct nf_conn *i, void *data); 2265 void *data; 2266 struct net *net; 2267 }; 2268 2269 static int iter_net_only(struct nf_conn *i, void *data) 2270 { 2271 struct iter_data *d = data; 2272 2273 if (!net_eq(d->net, nf_ct_net(i))) 2274 return 0; 2275 2276 return d->iter(i, d->data); 2277 } 2278 2279 static void 2280 __nf_ct_unconfirmed_destroy(struct net *net) 2281 { 2282 int cpu; 2283 2284 for_each_possible_cpu(cpu) { 2285 struct nf_conntrack_tuple_hash *h; 2286 struct hlist_nulls_node *n; 2287 struct ct_pcpu *pcpu; 2288 2289 pcpu = per_cpu_ptr(net->ct.pcpu_lists, cpu); 2290 2291 spin_lock_bh(&pcpu->lock); 2292 hlist_nulls_for_each_entry(h, n, &pcpu->unconfirmed, hnnode) { 2293 struct nf_conn *ct; 2294 2295 ct = nf_ct_tuplehash_to_ctrack(h); 2296 2297 /* we cannot call iter() on unconfirmed list, the 2298 * owning cpu can reallocate ct->ext at any time. 2299 */ 2300 set_bit(IPS_DYING_BIT, &ct->status); 2301 } 2302 spin_unlock_bh(&pcpu->lock); 2303 cond_resched(); 2304 } 2305 } 2306 2307 void nf_ct_unconfirmed_destroy(struct net *net) 2308 { 2309 might_sleep(); 2310 2311 if (atomic_read(&net->ct.count) > 0) { 2312 __nf_ct_unconfirmed_destroy(net); 2313 nf_queue_nf_hook_drop(net); 2314 synchronize_net(); 2315 } 2316 } 2317 EXPORT_SYMBOL_GPL(nf_ct_unconfirmed_destroy); 2318 2319 void nf_ct_iterate_cleanup_net(struct net *net, 2320 int (*iter)(struct nf_conn *i, void *data), 2321 void *data, u32 portid, int report) 2322 { 2323 struct iter_data d; 2324 2325 might_sleep(); 2326 2327 if (atomic_read(&net->ct.count) == 0) 2328 return; 2329 2330 d.iter = iter; 2331 d.data = data; 2332 d.net = net; 2333 2334 nf_ct_iterate_cleanup(iter_net_only, &d, portid, report); 2335 } 2336 EXPORT_SYMBOL_GPL(nf_ct_iterate_cleanup_net); 2337 2338 /** 2339 * nf_ct_iterate_destroy - destroy unconfirmed conntracks and iterate table 2340 * @iter: callback to invoke for each conntrack 2341 * @data: data to pass to @iter 2342 * 2343 * Like nf_ct_iterate_cleanup, but first marks conntracks on the 2344 * unconfirmed list as dying (so they will not be inserted into 2345 * main table). 2346 * 2347 * Can only be called in module exit path. 2348 */ 2349 void 2350 nf_ct_iterate_destroy(int (*iter)(struct nf_conn *i, void *data), void *data) 2351 { 2352 struct net *net; 2353 2354 down_read(&net_rwsem); 2355 for_each_net(net) { 2356 if (atomic_read(&net->ct.count) == 0) 2357 continue; 2358 __nf_ct_unconfirmed_destroy(net); 2359 nf_queue_nf_hook_drop(net); 2360 } 2361 up_read(&net_rwsem); 2362 2363 /* Need to wait for netns cleanup worker to finish, if its 2364 * running -- it might have deleted a net namespace from 2365 * the global list, so our __nf_ct_unconfirmed_destroy() might 2366 * not have affected all namespaces. 2367 */ 2368 net_ns_barrier(); 2369 2370 /* a conntrack could have been unlinked from unconfirmed list 2371 * before we grabbed pcpu lock in __nf_ct_unconfirmed_destroy(). 2372 * This makes sure its inserted into conntrack table. 2373 */ 2374 synchronize_net(); 2375 2376 nf_ct_iterate_cleanup(iter, data, 0, 0); 2377 } 2378 EXPORT_SYMBOL_GPL(nf_ct_iterate_destroy); 2379 2380 static int kill_all(struct nf_conn *i, void *data) 2381 { 2382 return net_eq(nf_ct_net(i), data); 2383 } 2384 2385 void nf_conntrack_cleanup_start(void) 2386 { 2387 conntrack_gc_work.exiting = true; 2388 RCU_INIT_POINTER(ip_ct_attach, NULL); 2389 } 2390 2391 void nf_conntrack_cleanup_end(void) 2392 { 2393 RCU_INIT_POINTER(nf_ct_hook, NULL); 2394 cancel_delayed_work_sync(&conntrack_gc_work.dwork); 2395 kvfree(nf_conntrack_hash); 2396 2397 nf_conntrack_proto_fini(); 2398 nf_conntrack_seqadj_fini(); 2399 nf_conntrack_labels_fini(); 2400 nf_conntrack_helper_fini(); 2401 nf_conntrack_timeout_fini(); 2402 nf_conntrack_ecache_fini(); 2403 nf_conntrack_tstamp_fini(); 2404 nf_conntrack_acct_fini(); 2405 nf_conntrack_expect_fini(); 2406 2407 kmem_cache_destroy(nf_conntrack_cachep); 2408 } 2409 2410 /* 2411 * Mishearing the voices in his head, our hero wonders how he's 2412 * supposed to kill the mall. 2413 */ 2414 void nf_conntrack_cleanup_net(struct net *net) 2415 { 2416 LIST_HEAD(single); 2417 2418 list_add(&net->exit_list, &single); 2419 nf_conntrack_cleanup_net_list(&single); 2420 } 2421 2422 void nf_conntrack_cleanup_net_list(struct list_head *net_exit_list) 2423 { 2424 int busy; 2425 struct net *net; 2426 2427 /* 2428 * This makes sure all current packets have passed through 2429 * netfilter framework. Roll on, two-stage module 2430 * delete... 2431 */ 2432 synchronize_net(); 2433 i_see_dead_people: 2434 busy = 0; 2435 list_for_each_entry(net, net_exit_list, exit_list) { 2436 nf_ct_iterate_cleanup(kill_all, net, 0, 0); 2437 if (atomic_read(&net->ct.count) != 0) 2438 busy = 1; 2439 } 2440 if (busy) { 2441 schedule(); 2442 goto i_see_dead_people; 2443 } 2444 2445 list_for_each_entry(net, net_exit_list, exit_list) { 2446 nf_conntrack_proto_pernet_fini(net); 2447 nf_conntrack_ecache_pernet_fini(net); 2448 nf_conntrack_expect_pernet_fini(net); 2449 free_percpu(net->ct.stat); 2450 free_percpu(net->ct.pcpu_lists); 2451 } 2452 } 2453 2454 void *nf_ct_alloc_hashtable(unsigned int *sizep, int nulls) 2455 { 2456 struct hlist_nulls_head *hash; 2457 unsigned int nr_slots, i; 2458 2459 if (*sizep > (UINT_MAX / sizeof(struct hlist_nulls_head))) 2460 return NULL; 2461 2462 BUILD_BUG_ON(sizeof(struct hlist_nulls_head) != sizeof(struct hlist_head)); 2463 nr_slots = *sizep = roundup(*sizep, PAGE_SIZE / sizeof(struct hlist_nulls_head)); 2464 2465 hash = kvcalloc(nr_slots, sizeof(struct hlist_nulls_head), GFP_KERNEL); 2466 2467 if (hash && nulls) 2468 for (i = 0; i < nr_slots; i++) 2469 INIT_HLIST_NULLS_HEAD(&hash[i], i); 2470 2471 return hash; 2472 } 2473 EXPORT_SYMBOL_GPL(nf_ct_alloc_hashtable); 2474 2475 int nf_conntrack_hash_resize(unsigned int hashsize) 2476 { 2477 int i, bucket; 2478 unsigned int old_size; 2479 struct hlist_nulls_head *hash, *old_hash; 2480 struct nf_conntrack_tuple_hash *h; 2481 struct nf_conn *ct; 2482 2483 if (!hashsize) 2484 return -EINVAL; 2485 2486 hash = nf_ct_alloc_hashtable(&hashsize, 1); 2487 if (!hash) 2488 return -ENOMEM; 2489 2490 old_size = nf_conntrack_htable_size; 2491 if (old_size == hashsize) { 2492 kvfree(hash); 2493 return 0; 2494 } 2495 2496 local_bh_disable(); 2497 nf_conntrack_all_lock(); 2498 write_seqcount_begin(&nf_conntrack_generation); 2499 2500 /* Lookups in the old hash might happen in parallel, which means we 2501 * might get false negatives during connection lookup. New connections 2502 * created because of a false negative won't make it into the hash 2503 * though since that required taking the locks. 2504 */ 2505 2506 for (i = 0; i < nf_conntrack_htable_size; i++) { 2507 while (!hlist_nulls_empty(&nf_conntrack_hash[i])) { 2508 h = hlist_nulls_entry(nf_conntrack_hash[i].first, 2509 struct nf_conntrack_tuple_hash, hnnode); 2510 ct = nf_ct_tuplehash_to_ctrack(h); 2511 hlist_nulls_del_rcu(&h->hnnode); 2512 bucket = __hash_conntrack(nf_ct_net(ct), 2513 &h->tuple, hashsize); 2514 hlist_nulls_add_head_rcu(&h->hnnode, &hash[bucket]); 2515 } 2516 } 2517 old_size = nf_conntrack_htable_size; 2518 old_hash = nf_conntrack_hash; 2519 2520 nf_conntrack_hash = hash; 2521 nf_conntrack_htable_size = hashsize; 2522 2523 write_seqcount_end(&nf_conntrack_generation); 2524 nf_conntrack_all_unlock(); 2525 local_bh_enable(); 2526 2527 synchronize_net(); 2528 kvfree(old_hash); 2529 return 0; 2530 } 2531 2532 int nf_conntrack_set_hashsize(const char *val, const struct kernel_param *kp) 2533 { 2534 unsigned int hashsize; 2535 int rc; 2536 2537 if (current->nsproxy->net_ns != &init_net) 2538 return -EOPNOTSUPP; 2539 2540 /* On boot, we can set this without any fancy locking. */ 2541 if (!nf_conntrack_hash) 2542 return param_set_uint(val, kp); 2543 2544 rc = kstrtouint(val, 0, &hashsize); 2545 if (rc) 2546 return rc; 2547 2548 return nf_conntrack_hash_resize(hashsize); 2549 } 2550 2551 static __always_inline unsigned int total_extension_size(void) 2552 { 2553 /* remember to add new extensions below */ 2554 BUILD_BUG_ON(NF_CT_EXT_NUM > 9); 2555 2556 return sizeof(struct nf_ct_ext) + 2557 sizeof(struct nf_conn_help) 2558 #if IS_ENABLED(CONFIG_NF_NAT) 2559 + sizeof(struct nf_conn_nat) 2560 #endif 2561 + sizeof(struct nf_conn_seqadj) 2562 + sizeof(struct nf_conn_acct) 2563 #ifdef CONFIG_NF_CONNTRACK_EVENTS 2564 + sizeof(struct nf_conntrack_ecache) 2565 #endif 2566 #ifdef CONFIG_NF_CONNTRACK_TIMESTAMP 2567 + sizeof(struct nf_conn_tstamp) 2568 #endif 2569 #ifdef CONFIG_NF_CONNTRACK_TIMEOUT 2570 + sizeof(struct nf_conn_timeout) 2571 #endif 2572 #ifdef CONFIG_NF_CONNTRACK_LABELS 2573 + sizeof(struct nf_conn_labels) 2574 #endif 2575 #if IS_ENABLED(CONFIG_NETFILTER_SYNPROXY) 2576 + sizeof(struct nf_conn_synproxy) 2577 #endif 2578 ; 2579 }; 2580 2581 int nf_conntrack_init_start(void) 2582 { 2583 unsigned long nr_pages = totalram_pages(); 2584 int max_factor = 8; 2585 int ret = -ENOMEM; 2586 int i; 2587 2588 /* struct nf_ct_ext uses u8 to store offsets/size */ 2589 BUILD_BUG_ON(total_extension_size() > 255u); 2590 2591 seqcount_spinlock_init(&nf_conntrack_generation, 2592 &nf_conntrack_locks_all_lock); 2593 2594 for (i = 0; i < CONNTRACK_LOCKS; i++) 2595 spin_lock_init(&nf_conntrack_locks[i]); 2596 2597 if (!nf_conntrack_htable_size) { 2598 /* Idea from tcp.c: use 1/16384 of memory. 2599 * On i386: 32MB machine has 512 buckets. 2600 * >= 1GB machines have 16384 buckets. 2601 * >= 4GB machines have 65536 buckets. 2602 */ 2603 nf_conntrack_htable_size 2604 = (((nr_pages << PAGE_SHIFT) / 16384) 2605 / sizeof(struct hlist_head)); 2606 if (nr_pages > (4 * (1024 * 1024 * 1024 / PAGE_SIZE))) 2607 nf_conntrack_htable_size = 65536; 2608 else if (nr_pages > (1024 * 1024 * 1024 / PAGE_SIZE)) 2609 nf_conntrack_htable_size = 16384; 2610 if (nf_conntrack_htable_size < 32) 2611 nf_conntrack_htable_size = 32; 2612 2613 /* Use a max. factor of four by default to get the same max as 2614 * with the old struct list_heads. When a table size is given 2615 * we use the old value of 8 to avoid reducing the max. 2616 * entries. */ 2617 max_factor = 4; 2618 } 2619 2620 nf_conntrack_hash = nf_ct_alloc_hashtable(&nf_conntrack_htable_size, 1); 2621 if (!nf_conntrack_hash) 2622 return -ENOMEM; 2623 2624 nf_conntrack_max = max_factor * nf_conntrack_htable_size; 2625 2626 nf_conntrack_cachep = kmem_cache_create("nf_conntrack", 2627 sizeof(struct nf_conn), 2628 NFCT_INFOMASK + 1, 2629 SLAB_TYPESAFE_BY_RCU | SLAB_HWCACHE_ALIGN, NULL); 2630 if (!nf_conntrack_cachep) 2631 goto err_cachep; 2632 2633 ret = nf_conntrack_expect_init(); 2634 if (ret < 0) 2635 goto err_expect; 2636 2637 ret = nf_conntrack_acct_init(); 2638 if (ret < 0) 2639 goto err_acct; 2640 2641 ret = nf_conntrack_tstamp_init(); 2642 if (ret < 0) 2643 goto err_tstamp; 2644 2645 ret = nf_conntrack_ecache_init(); 2646 if (ret < 0) 2647 goto err_ecache; 2648 2649 ret = nf_conntrack_timeout_init(); 2650 if (ret < 0) 2651 goto err_timeout; 2652 2653 ret = nf_conntrack_helper_init(); 2654 if (ret < 0) 2655 goto err_helper; 2656 2657 ret = nf_conntrack_labels_init(); 2658 if (ret < 0) 2659 goto err_labels; 2660 2661 ret = nf_conntrack_seqadj_init(); 2662 if (ret < 0) 2663 goto err_seqadj; 2664 2665 ret = nf_conntrack_proto_init(); 2666 if (ret < 0) 2667 goto err_proto; 2668 2669 conntrack_gc_work_init(&conntrack_gc_work); 2670 queue_delayed_work(system_power_efficient_wq, &conntrack_gc_work.dwork, HZ); 2671 2672 return 0; 2673 2674 err_proto: 2675 nf_conntrack_seqadj_fini(); 2676 err_seqadj: 2677 nf_conntrack_labels_fini(); 2678 err_labels: 2679 nf_conntrack_helper_fini(); 2680 err_helper: 2681 nf_conntrack_timeout_fini(); 2682 err_timeout: 2683 nf_conntrack_ecache_fini(); 2684 err_ecache: 2685 nf_conntrack_tstamp_fini(); 2686 err_tstamp: 2687 nf_conntrack_acct_fini(); 2688 err_acct: 2689 nf_conntrack_expect_fini(); 2690 err_expect: 2691 kmem_cache_destroy(nf_conntrack_cachep); 2692 err_cachep: 2693 kvfree(nf_conntrack_hash); 2694 return ret; 2695 } 2696 2697 static struct nf_ct_hook nf_conntrack_hook = { 2698 .update = nf_conntrack_update, 2699 .destroy = destroy_conntrack, 2700 .get_tuple_skb = nf_conntrack_get_tuple_skb, 2701 }; 2702 2703 void nf_conntrack_init_end(void) 2704 { 2705 /* For use by REJECT target */ 2706 RCU_INIT_POINTER(ip_ct_attach, nf_conntrack_attach); 2707 RCU_INIT_POINTER(nf_ct_hook, &nf_conntrack_hook); 2708 } 2709 2710 /* 2711 * We need to use special "null" values, not used in hash table 2712 */ 2713 #define UNCONFIRMED_NULLS_VAL ((1<<30)+0) 2714 #define DYING_NULLS_VAL ((1<<30)+1) 2715 2716 int nf_conntrack_init_net(struct net *net) 2717 { 2718 int ret = -ENOMEM; 2719 int cpu; 2720 2721 BUILD_BUG_ON(IP_CT_UNTRACKED == IP_CT_NUMBER); 2722 BUILD_BUG_ON_NOT_POWER_OF_2(CONNTRACK_LOCKS); 2723 atomic_set(&net->ct.count, 0); 2724 2725 net->ct.pcpu_lists = alloc_percpu(struct ct_pcpu); 2726 if (!net->ct.pcpu_lists) 2727 goto err_stat; 2728 2729 for_each_possible_cpu(cpu) { 2730 struct ct_pcpu *pcpu = per_cpu_ptr(net->ct.pcpu_lists, cpu); 2731 2732 spin_lock_init(&pcpu->lock); 2733 INIT_HLIST_NULLS_HEAD(&pcpu->unconfirmed, UNCONFIRMED_NULLS_VAL); 2734 INIT_HLIST_NULLS_HEAD(&pcpu->dying, DYING_NULLS_VAL); 2735 } 2736 2737 net->ct.stat = alloc_percpu(struct ip_conntrack_stat); 2738 if (!net->ct.stat) 2739 goto err_pcpu_lists; 2740 2741 ret = nf_conntrack_expect_pernet_init(net); 2742 if (ret < 0) 2743 goto err_expect; 2744 2745 nf_conntrack_acct_pernet_init(net); 2746 nf_conntrack_tstamp_pernet_init(net); 2747 nf_conntrack_ecache_pernet_init(net); 2748 nf_conntrack_helper_pernet_init(net); 2749 nf_conntrack_proto_pernet_init(net); 2750 2751 return 0; 2752 2753 err_expect: 2754 free_percpu(net->ct.stat); 2755 err_pcpu_lists: 2756 free_percpu(net->ct.pcpu_lists); 2757 err_stat: 2758 return ret; 2759 } 2760