1 // SPDX-License-Identifier: GPL-2.0-only 2 /* Connection state tracking for netfilter. This is separated from, 3 but required by, the NAT layer; it can also be used by an iptables 4 extension. */ 5 6 /* (C) 1999-2001 Paul `Rusty' Russell 7 * (C) 2002-2006 Netfilter Core Team <coreteam@netfilter.org> 8 * (C) 2003,2004 USAGI/WIDE Project <http://www.linux-ipv6.org> 9 * (C) 2005-2012 Patrick McHardy <kaber@trash.net> 10 */ 11 12 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 13 14 #include <linux/types.h> 15 #include <linux/netfilter.h> 16 #include <linux/module.h> 17 #include <linux/sched.h> 18 #include <linux/skbuff.h> 19 #include <linux/proc_fs.h> 20 #include <linux/vmalloc.h> 21 #include <linux/stddef.h> 22 #include <linux/slab.h> 23 #include <linux/random.h> 24 #include <linux/jhash.h> 25 #include <linux/siphash.h> 26 #include <linux/err.h> 27 #include <linux/percpu.h> 28 #include <linux/moduleparam.h> 29 #include <linux/notifier.h> 30 #include <linux/kernel.h> 31 #include <linux/netdevice.h> 32 #include <linux/socket.h> 33 #include <linux/mm.h> 34 #include <linux/nsproxy.h> 35 #include <linux/rculist_nulls.h> 36 37 #include <net/netfilter/nf_conntrack.h> 38 #include <net/netfilter/nf_conntrack_l4proto.h> 39 #include <net/netfilter/nf_conntrack_expect.h> 40 #include <net/netfilter/nf_conntrack_helper.h> 41 #include <net/netfilter/nf_conntrack_seqadj.h> 42 #include <net/netfilter/nf_conntrack_core.h> 43 #include <net/netfilter/nf_conntrack_extend.h> 44 #include <net/netfilter/nf_conntrack_acct.h> 45 #include <net/netfilter/nf_conntrack_ecache.h> 46 #include <net/netfilter/nf_conntrack_zones.h> 47 #include <net/netfilter/nf_conntrack_timestamp.h> 48 #include <net/netfilter/nf_conntrack_timeout.h> 49 #include <net/netfilter/nf_conntrack_labels.h> 50 #include <net/netfilter/nf_conntrack_synproxy.h> 51 #include <net/netfilter/nf_nat.h> 52 #include <net/netfilter/nf_nat_helper.h> 53 #include <net/netns/hash.h> 54 #include <net/ip.h> 55 56 #include "nf_internals.h" 57 58 __cacheline_aligned_in_smp spinlock_t nf_conntrack_locks[CONNTRACK_LOCKS]; 59 EXPORT_SYMBOL_GPL(nf_conntrack_locks); 60 61 __cacheline_aligned_in_smp DEFINE_SPINLOCK(nf_conntrack_expect_lock); 62 EXPORT_SYMBOL_GPL(nf_conntrack_expect_lock); 63 64 struct hlist_nulls_head *nf_conntrack_hash __read_mostly; 65 EXPORT_SYMBOL_GPL(nf_conntrack_hash); 66 67 struct conntrack_gc_work { 68 struct delayed_work dwork; 69 u32 last_bucket; 70 bool exiting; 71 bool early_drop; 72 long next_gc_run; 73 }; 74 75 static __read_mostly struct kmem_cache *nf_conntrack_cachep; 76 static DEFINE_SPINLOCK(nf_conntrack_locks_all_lock); 77 static __read_mostly bool nf_conntrack_locks_all; 78 79 /* every gc cycle scans at most 1/GC_MAX_BUCKETS_DIV part of table */ 80 #define GC_MAX_BUCKETS_DIV 128u 81 /* upper bound of full table scan */ 82 #define GC_MAX_SCAN_JIFFIES (16u * HZ) 83 /* desired ratio of entries found to be expired */ 84 #define GC_EVICT_RATIO 50u 85 86 static struct conntrack_gc_work conntrack_gc_work; 87 88 void nf_conntrack_lock(spinlock_t *lock) __acquires(lock) 89 { 90 /* 1) Acquire the lock */ 91 spin_lock(lock); 92 93 /* 2) read nf_conntrack_locks_all, with ACQUIRE semantics 94 * It pairs with the smp_store_release() in nf_conntrack_all_unlock() 95 */ 96 if (likely(smp_load_acquire(&nf_conntrack_locks_all) == false)) 97 return; 98 99 /* fast path failed, unlock */ 100 spin_unlock(lock); 101 102 /* Slow path 1) get global lock */ 103 spin_lock(&nf_conntrack_locks_all_lock); 104 105 /* Slow path 2) get the lock we want */ 106 spin_lock(lock); 107 108 /* Slow path 3) release the global lock */ 109 spin_unlock(&nf_conntrack_locks_all_lock); 110 } 111 EXPORT_SYMBOL_GPL(nf_conntrack_lock); 112 113 static void nf_conntrack_double_unlock(unsigned int h1, unsigned int h2) 114 { 115 h1 %= CONNTRACK_LOCKS; 116 h2 %= CONNTRACK_LOCKS; 117 spin_unlock(&nf_conntrack_locks[h1]); 118 if (h1 != h2) 119 spin_unlock(&nf_conntrack_locks[h2]); 120 } 121 122 /* return true if we need to recompute hashes (in case hash table was resized) */ 123 static bool nf_conntrack_double_lock(struct net *net, unsigned int h1, 124 unsigned int h2, unsigned int sequence) 125 { 126 h1 %= CONNTRACK_LOCKS; 127 h2 %= CONNTRACK_LOCKS; 128 if (h1 <= h2) { 129 nf_conntrack_lock(&nf_conntrack_locks[h1]); 130 if (h1 != h2) 131 spin_lock_nested(&nf_conntrack_locks[h2], 132 SINGLE_DEPTH_NESTING); 133 } else { 134 nf_conntrack_lock(&nf_conntrack_locks[h2]); 135 spin_lock_nested(&nf_conntrack_locks[h1], 136 SINGLE_DEPTH_NESTING); 137 } 138 if (read_seqcount_retry(&nf_conntrack_generation, sequence)) { 139 nf_conntrack_double_unlock(h1, h2); 140 return true; 141 } 142 return false; 143 } 144 145 static void nf_conntrack_all_lock(void) 146 __acquires(&nf_conntrack_locks_all_lock) 147 { 148 int i; 149 150 spin_lock(&nf_conntrack_locks_all_lock); 151 152 nf_conntrack_locks_all = true; 153 154 for (i = 0; i < CONNTRACK_LOCKS; i++) { 155 spin_lock(&nf_conntrack_locks[i]); 156 157 /* This spin_unlock provides the "release" to ensure that 158 * nf_conntrack_locks_all==true is visible to everyone that 159 * acquired spin_lock(&nf_conntrack_locks[]). 160 */ 161 spin_unlock(&nf_conntrack_locks[i]); 162 } 163 } 164 165 static void nf_conntrack_all_unlock(void) 166 __releases(&nf_conntrack_locks_all_lock) 167 { 168 /* All prior stores must be complete before we clear 169 * 'nf_conntrack_locks_all'. Otherwise nf_conntrack_lock() 170 * might observe the false value but not the entire 171 * critical section. 172 * It pairs with the smp_load_acquire() in nf_conntrack_lock() 173 */ 174 smp_store_release(&nf_conntrack_locks_all, false); 175 spin_unlock(&nf_conntrack_locks_all_lock); 176 } 177 178 unsigned int nf_conntrack_htable_size __read_mostly; 179 EXPORT_SYMBOL_GPL(nf_conntrack_htable_size); 180 181 unsigned int nf_conntrack_max __read_mostly; 182 EXPORT_SYMBOL_GPL(nf_conntrack_max); 183 seqcount_spinlock_t nf_conntrack_generation __read_mostly; 184 static unsigned int nf_conntrack_hash_rnd __read_mostly; 185 186 static u32 hash_conntrack_raw(const struct nf_conntrack_tuple *tuple, 187 const struct net *net) 188 { 189 unsigned int n; 190 u32 seed; 191 192 get_random_once(&nf_conntrack_hash_rnd, sizeof(nf_conntrack_hash_rnd)); 193 194 /* The direction must be ignored, so we hash everything up to the 195 * destination ports (which is a multiple of 4) and treat the last 196 * three bytes manually. 197 */ 198 seed = nf_conntrack_hash_rnd ^ net_hash_mix(net); 199 n = (sizeof(tuple->src) + sizeof(tuple->dst.u3)) / sizeof(u32); 200 return jhash2((u32 *)tuple, n, seed ^ 201 (((__force __u16)tuple->dst.u.all << 16) | 202 tuple->dst.protonum)); 203 } 204 205 static u32 scale_hash(u32 hash) 206 { 207 return reciprocal_scale(hash, nf_conntrack_htable_size); 208 } 209 210 static u32 __hash_conntrack(const struct net *net, 211 const struct nf_conntrack_tuple *tuple, 212 unsigned int size) 213 { 214 return reciprocal_scale(hash_conntrack_raw(tuple, net), size); 215 } 216 217 static u32 hash_conntrack(const struct net *net, 218 const struct nf_conntrack_tuple *tuple) 219 { 220 return scale_hash(hash_conntrack_raw(tuple, net)); 221 } 222 223 static bool nf_ct_get_tuple_ports(const struct sk_buff *skb, 224 unsigned int dataoff, 225 struct nf_conntrack_tuple *tuple) 226 { struct { 227 __be16 sport; 228 __be16 dport; 229 } _inet_hdr, *inet_hdr; 230 231 /* Actually only need first 4 bytes to get ports. */ 232 inet_hdr = skb_header_pointer(skb, dataoff, sizeof(_inet_hdr), &_inet_hdr); 233 if (!inet_hdr) 234 return false; 235 236 tuple->src.u.udp.port = inet_hdr->sport; 237 tuple->dst.u.udp.port = inet_hdr->dport; 238 return true; 239 } 240 241 static bool 242 nf_ct_get_tuple(const struct sk_buff *skb, 243 unsigned int nhoff, 244 unsigned int dataoff, 245 u_int16_t l3num, 246 u_int8_t protonum, 247 struct net *net, 248 struct nf_conntrack_tuple *tuple) 249 { 250 unsigned int size; 251 const __be32 *ap; 252 __be32 _addrs[8]; 253 254 memset(tuple, 0, sizeof(*tuple)); 255 256 tuple->src.l3num = l3num; 257 switch (l3num) { 258 case NFPROTO_IPV4: 259 nhoff += offsetof(struct iphdr, saddr); 260 size = 2 * sizeof(__be32); 261 break; 262 case NFPROTO_IPV6: 263 nhoff += offsetof(struct ipv6hdr, saddr); 264 size = sizeof(_addrs); 265 break; 266 default: 267 return true; 268 } 269 270 ap = skb_header_pointer(skb, nhoff, size, _addrs); 271 if (!ap) 272 return false; 273 274 switch (l3num) { 275 case NFPROTO_IPV4: 276 tuple->src.u3.ip = ap[0]; 277 tuple->dst.u3.ip = ap[1]; 278 break; 279 case NFPROTO_IPV6: 280 memcpy(tuple->src.u3.ip6, ap, sizeof(tuple->src.u3.ip6)); 281 memcpy(tuple->dst.u3.ip6, ap + 4, sizeof(tuple->dst.u3.ip6)); 282 break; 283 } 284 285 tuple->dst.protonum = protonum; 286 tuple->dst.dir = IP_CT_DIR_ORIGINAL; 287 288 switch (protonum) { 289 #if IS_ENABLED(CONFIG_IPV6) 290 case IPPROTO_ICMPV6: 291 return icmpv6_pkt_to_tuple(skb, dataoff, net, tuple); 292 #endif 293 case IPPROTO_ICMP: 294 return icmp_pkt_to_tuple(skb, dataoff, net, tuple); 295 #ifdef CONFIG_NF_CT_PROTO_GRE 296 case IPPROTO_GRE: 297 return gre_pkt_to_tuple(skb, dataoff, net, tuple); 298 #endif 299 case IPPROTO_TCP: 300 case IPPROTO_UDP: /* fallthrough */ 301 return nf_ct_get_tuple_ports(skb, dataoff, tuple); 302 #ifdef CONFIG_NF_CT_PROTO_UDPLITE 303 case IPPROTO_UDPLITE: 304 return nf_ct_get_tuple_ports(skb, dataoff, tuple); 305 #endif 306 #ifdef CONFIG_NF_CT_PROTO_SCTP 307 case IPPROTO_SCTP: 308 return nf_ct_get_tuple_ports(skb, dataoff, tuple); 309 #endif 310 #ifdef CONFIG_NF_CT_PROTO_DCCP 311 case IPPROTO_DCCP: 312 return nf_ct_get_tuple_ports(skb, dataoff, tuple); 313 #endif 314 default: 315 break; 316 } 317 318 return true; 319 } 320 321 static int ipv4_get_l4proto(const struct sk_buff *skb, unsigned int nhoff, 322 u_int8_t *protonum) 323 { 324 int dataoff = -1; 325 const struct iphdr *iph; 326 struct iphdr _iph; 327 328 iph = skb_header_pointer(skb, nhoff, sizeof(_iph), &_iph); 329 if (!iph) 330 return -1; 331 332 /* Conntrack defragments packets, we might still see fragments 333 * inside ICMP packets though. 334 */ 335 if (iph->frag_off & htons(IP_OFFSET)) 336 return -1; 337 338 dataoff = nhoff + (iph->ihl << 2); 339 *protonum = iph->protocol; 340 341 /* Check bogus IP headers */ 342 if (dataoff > skb->len) { 343 pr_debug("bogus IPv4 packet: nhoff %u, ihl %u, skblen %u\n", 344 nhoff, iph->ihl << 2, skb->len); 345 return -1; 346 } 347 return dataoff; 348 } 349 350 #if IS_ENABLED(CONFIG_IPV6) 351 static int ipv6_get_l4proto(const struct sk_buff *skb, unsigned int nhoff, 352 u8 *protonum) 353 { 354 int protoff = -1; 355 unsigned int extoff = nhoff + sizeof(struct ipv6hdr); 356 __be16 frag_off; 357 u8 nexthdr; 358 359 if (skb_copy_bits(skb, nhoff + offsetof(struct ipv6hdr, nexthdr), 360 &nexthdr, sizeof(nexthdr)) != 0) { 361 pr_debug("can't get nexthdr\n"); 362 return -1; 363 } 364 protoff = ipv6_skip_exthdr(skb, extoff, &nexthdr, &frag_off); 365 /* 366 * (protoff == skb->len) means the packet has not data, just 367 * IPv6 and possibly extensions headers, but it is tracked anyway 368 */ 369 if (protoff < 0 || (frag_off & htons(~0x7)) != 0) { 370 pr_debug("can't find proto in pkt\n"); 371 return -1; 372 } 373 374 *protonum = nexthdr; 375 return protoff; 376 } 377 #endif 378 379 static int get_l4proto(const struct sk_buff *skb, 380 unsigned int nhoff, u8 pf, u8 *l4num) 381 { 382 switch (pf) { 383 case NFPROTO_IPV4: 384 return ipv4_get_l4proto(skb, nhoff, l4num); 385 #if IS_ENABLED(CONFIG_IPV6) 386 case NFPROTO_IPV6: 387 return ipv6_get_l4proto(skb, nhoff, l4num); 388 #endif 389 default: 390 *l4num = 0; 391 break; 392 } 393 return -1; 394 } 395 396 bool nf_ct_get_tuplepr(const struct sk_buff *skb, unsigned int nhoff, 397 u_int16_t l3num, 398 struct net *net, struct nf_conntrack_tuple *tuple) 399 { 400 u8 protonum; 401 int protoff; 402 403 protoff = get_l4proto(skb, nhoff, l3num, &protonum); 404 if (protoff <= 0) 405 return false; 406 407 return nf_ct_get_tuple(skb, nhoff, protoff, l3num, protonum, net, tuple); 408 } 409 EXPORT_SYMBOL_GPL(nf_ct_get_tuplepr); 410 411 bool 412 nf_ct_invert_tuple(struct nf_conntrack_tuple *inverse, 413 const struct nf_conntrack_tuple *orig) 414 { 415 memset(inverse, 0, sizeof(*inverse)); 416 417 inverse->src.l3num = orig->src.l3num; 418 419 switch (orig->src.l3num) { 420 case NFPROTO_IPV4: 421 inverse->src.u3.ip = orig->dst.u3.ip; 422 inverse->dst.u3.ip = orig->src.u3.ip; 423 break; 424 case NFPROTO_IPV6: 425 inverse->src.u3.in6 = orig->dst.u3.in6; 426 inverse->dst.u3.in6 = orig->src.u3.in6; 427 break; 428 default: 429 break; 430 } 431 432 inverse->dst.dir = !orig->dst.dir; 433 434 inverse->dst.protonum = orig->dst.protonum; 435 436 switch (orig->dst.protonum) { 437 case IPPROTO_ICMP: 438 return nf_conntrack_invert_icmp_tuple(inverse, orig); 439 #if IS_ENABLED(CONFIG_IPV6) 440 case IPPROTO_ICMPV6: 441 return nf_conntrack_invert_icmpv6_tuple(inverse, orig); 442 #endif 443 } 444 445 inverse->src.u.all = orig->dst.u.all; 446 inverse->dst.u.all = orig->src.u.all; 447 return true; 448 } 449 EXPORT_SYMBOL_GPL(nf_ct_invert_tuple); 450 451 /* Generate a almost-unique pseudo-id for a given conntrack. 452 * 453 * intentionally doesn't re-use any of the seeds used for hash 454 * table location, we assume id gets exposed to userspace. 455 * 456 * Following nf_conn items do not change throughout lifetime 457 * of the nf_conn: 458 * 459 * 1. nf_conn address 460 * 2. nf_conn->master address (normally NULL) 461 * 3. the associated net namespace 462 * 4. the original direction tuple 463 */ 464 u32 nf_ct_get_id(const struct nf_conn *ct) 465 { 466 static __read_mostly siphash_key_t ct_id_seed; 467 unsigned long a, b, c, d; 468 469 net_get_random_once(&ct_id_seed, sizeof(ct_id_seed)); 470 471 a = (unsigned long)ct; 472 b = (unsigned long)ct->master; 473 c = (unsigned long)nf_ct_net(ct); 474 d = (unsigned long)siphash(&ct->tuplehash[IP_CT_DIR_ORIGINAL].tuple, 475 sizeof(ct->tuplehash[IP_CT_DIR_ORIGINAL].tuple), 476 &ct_id_seed); 477 #ifdef CONFIG_64BIT 478 return siphash_4u64((u64)a, (u64)b, (u64)c, (u64)d, &ct_id_seed); 479 #else 480 return siphash_4u32((u32)a, (u32)b, (u32)c, (u32)d, &ct_id_seed); 481 #endif 482 } 483 EXPORT_SYMBOL_GPL(nf_ct_get_id); 484 485 static void 486 clean_from_lists(struct nf_conn *ct) 487 { 488 pr_debug("clean_from_lists(%p)\n", ct); 489 hlist_nulls_del_rcu(&ct->tuplehash[IP_CT_DIR_ORIGINAL].hnnode); 490 hlist_nulls_del_rcu(&ct->tuplehash[IP_CT_DIR_REPLY].hnnode); 491 492 /* Destroy all pending expectations */ 493 nf_ct_remove_expectations(ct); 494 } 495 496 /* must be called with local_bh_disable */ 497 static void nf_ct_add_to_dying_list(struct nf_conn *ct) 498 { 499 struct ct_pcpu *pcpu; 500 501 /* add this conntrack to the (per cpu) dying list */ 502 ct->cpu = smp_processor_id(); 503 pcpu = per_cpu_ptr(nf_ct_net(ct)->ct.pcpu_lists, ct->cpu); 504 505 spin_lock(&pcpu->lock); 506 hlist_nulls_add_head(&ct->tuplehash[IP_CT_DIR_ORIGINAL].hnnode, 507 &pcpu->dying); 508 spin_unlock(&pcpu->lock); 509 } 510 511 /* must be called with local_bh_disable */ 512 static void nf_ct_add_to_unconfirmed_list(struct nf_conn *ct) 513 { 514 struct ct_pcpu *pcpu; 515 516 /* add this conntrack to the (per cpu) unconfirmed list */ 517 ct->cpu = smp_processor_id(); 518 pcpu = per_cpu_ptr(nf_ct_net(ct)->ct.pcpu_lists, ct->cpu); 519 520 spin_lock(&pcpu->lock); 521 hlist_nulls_add_head(&ct->tuplehash[IP_CT_DIR_ORIGINAL].hnnode, 522 &pcpu->unconfirmed); 523 spin_unlock(&pcpu->lock); 524 } 525 526 /* must be called with local_bh_disable */ 527 static void nf_ct_del_from_dying_or_unconfirmed_list(struct nf_conn *ct) 528 { 529 struct ct_pcpu *pcpu; 530 531 /* We overload first tuple to link into unconfirmed or dying list.*/ 532 pcpu = per_cpu_ptr(nf_ct_net(ct)->ct.pcpu_lists, ct->cpu); 533 534 spin_lock(&pcpu->lock); 535 BUG_ON(hlist_nulls_unhashed(&ct->tuplehash[IP_CT_DIR_ORIGINAL].hnnode)); 536 hlist_nulls_del_rcu(&ct->tuplehash[IP_CT_DIR_ORIGINAL].hnnode); 537 spin_unlock(&pcpu->lock); 538 } 539 540 #define NFCT_ALIGN(len) (((len) + NFCT_INFOMASK) & ~NFCT_INFOMASK) 541 542 /* Released via destroy_conntrack() */ 543 struct nf_conn *nf_ct_tmpl_alloc(struct net *net, 544 const struct nf_conntrack_zone *zone, 545 gfp_t flags) 546 { 547 struct nf_conn *tmpl, *p; 548 549 if (ARCH_KMALLOC_MINALIGN <= NFCT_INFOMASK) { 550 tmpl = kzalloc(sizeof(*tmpl) + NFCT_INFOMASK, flags); 551 if (!tmpl) 552 return NULL; 553 554 p = tmpl; 555 tmpl = (struct nf_conn *)NFCT_ALIGN((unsigned long)p); 556 if (tmpl != p) { 557 tmpl = (struct nf_conn *)NFCT_ALIGN((unsigned long)p); 558 tmpl->proto.tmpl_padto = (char *)tmpl - (char *)p; 559 } 560 } else { 561 tmpl = kzalloc(sizeof(*tmpl), flags); 562 if (!tmpl) 563 return NULL; 564 } 565 566 tmpl->status = IPS_TEMPLATE; 567 write_pnet(&tmpl->ct_net, net); 568 nf_ct_zone_add(tmpl, zone); 569 atomic_set(&tmpl->ct_general.use, 0); 570 571 return tmpl; 572 } 573 EXPORT_SYMBOL_GPL(nf_ct_tmpl_alloc); 574 575 void nf_ct_tmpl_free(struct nf_conn *tmpl) 576 { 577 nf_ct_ext_destroy(tmpl); 578 579 if (ARCH_KMALLOC_MINALIGN <= NFCT_INFOMASK) 580 kfree((char *)tmpl - tmpl->proto.tmpl_padto); 581 else 582 kfree(tmpl); 583 } 584 EXPORT_SYMBOL_GPL(nf_ct_tmpl_free); 585 586 static void destroy_gre_conntrack(struct nf_conn *ct) 587 { 588 #ifdef CONFIG_NF_CT_PROTO_GRE 589 struct nf_conn *master = ct->master; 590 591 if (master) 592 nf_ct_gre_keymap_destroy(master); 593 #endif 594 } 595 596 static void 597 destroy_conntrack(struct nf_conntrack *nfct) 598 { 599 struct nf_conn *ct = (struct nf_conn *)nfct; 600 601 pr_debug("destroy_conntrack(%p)\n", ct); 602 WARN_ON(atomic_read(&nfct->use) != 0); 603 604 if (unlikely(nf_ct_is_template(ct))) { 605 nf_ct_tmpl_free(ct); 606 return; 607 } 608 609 if (unlikely(nf_ct_protonum(ct) == IPPROTO_GRE)) 610 destroy_gre_conntrack(ct); 611 612 local_bh_disable(); 613 /* Expectations will have been removed in clean_from_lists, 614 * except TFTP can create an expectation on the first packet, 615 * before connection is in the list, so we need to clean here, 616 * too. 617 */ 618 nf_ct_remove_expectations(ct); 619 620 nf_ct_del_from_dying_or_unconfirmed_list(ct); 621 622 local_bh_enable(); 623 624 if (ct->master) 625 nf_ct_put(ct->master); 626 627 pr_debug("destroy_conntrack: returning ct=%p to slab\n", ct); 628 nf_conntrack_free(ct); 629 } 630 631 static void nf_ct_delete_from_lists(struct nf_conn *ct) 632 { 633 struct net *net = nf_ct_net(ct); 634 unsigned int hash, reply_hash; 635 unsigned int sequence; 636 637 nf_ct_helper_destroy(ct); 638 639 local_bh_disable(); 640 do { 641 sequence = read_seqcount_begin(&nf_conntrack_generation); 642 hash = hash_conntrack(net, 643 &ct->tuplehash[IP_CT_DIR_ORIGINAL].tuple); 644 reply_hash = hash_conntrack(net, 645 &ct->tuplehash[IP_CT_DIR_REPLY].tuple); 646 } while (nf_conntrack_double_lock(net, hash, reply_hash, sequence)); 647 648 clean_from_lists(ct); 649 nf_conntrack_double_unlock(hash, reply_hash); 650 651 nf_ct_add_to_dying_list(ct); 652 653 local_bh_enable(); 654 } 655 656 bool nf_ct_delete(struct nf_conn *ct, u32 portid, int report) 657 { 658 struct nf_conn_tstamp *tstamp; 659 struct net *net; 660 661 if (test_and_set_bit(IPS_DYING_BIT, &ct->status)) 662 return false; 663 664 tstamp = nf_conn_tstamp_find(ct); 665 if (tstamp && tstamp->stop == 0) 666 tstamp->stop = ktime_get_real_ns(); 667 668 if (nf_conntrack_event_report(IPCT_DESTROY, ct, 669 portid, report) < 0) { 670 /* destroy event was not delivered. nf_ct_put will 671 * be done by event cache worker on redelivery. 672 */ 673 nf_ct_delete_from_lists(ct); 674 nf_conntrack_ecache_work(nf_ct_net(ct), NFCT_ECACHE_DESTROY_FAIL); 675 return false; 676 } 677 678 net = nf_ct_net(ct); 679 if (nf_conntrack_ecache_dwork_pending(net)) 680 nf_conntrack_ecache_work(net, NFCT_ECACHE_DESTROY_SENT); 681 nf_ct_delete_from_lists(ct); 682 nf_ct_put(ct); 683 return true; 684 } 685 EXPORT_SYMBOL_GPL(nf_ct_delete); 686 687 static inline bool 688 nf_ct_key_equal(struct nf_conntrack_tuple_hash *h, 689 const struct nf_conntrack_tuple *tuple, 690 const struct nf_conntrack_zone *zone, 691 const struct net *net) 692 { 693 struct nf_conn *ct = nf_ct_tuplehash_to_ctrack(h); 694 695 /* A conntrack can be recreated with the equal tuple, 696 * so we need to check that the conntrack is confirmed 697 */ 698 return nf_ct_tuple_equal(tuple, &h->tuple) && 699 nf_ct_zone_equal(ct, zone, NF_CT_DIRECTION(h)) && 700 nf_ct_is_confirmed(ct) && 701 net_eq(net, nf_ct_net(ct)); 702 } 703 704 static inline bool 705 nf_ct_match(const struct nf_conn *ct1, const struct nf_conn *ct2) 706 { 707 return nf_ct_tuple_equal(&ct1->tuplehash[IP_CT_DIR_ORIGINAL].tuple, 708 &ct2->tuplehash[IP_CT_DIR_ORIGINAL].tuple) && 709 nf_ct_tuple_equal(&ct1->tuplehash[IP_CT_DIR_REPLY].tuple, 710 &ct2->tuplehash[IP_CT_DIR_REPLY].tuple) && 711 nf_ct_zone_equal(ct1, nf_ct_zone(ct2), IP_CT_DIR_ORIGINAL) && 712 nf_ct_zone_equal(ct1, nf_ct_zone(ct2), IP_CT_DIR_REPLY) && 713 net_eq(nf_ct_net(ct1), nf_ct_net(ct2)); 714 } 715 716 /* caller must hold rcu readlock and none of the nf_conntrack_locks */ 717 static void nf_ct_gc_expired(struct nf_conn *ct) 718 { 719 if (!atomic_inc_not_zero(&ct->ct_general.use)) 720 return; 721 722 if (nf_ct_should_gc(ct)) 723 nf_ct_kill(ct); 724 725 nf_ct_put(ct); 726 } 727 728 /* 729 * Warning : 730 * - Caller must take a reference on returned object 731 * and recheck nf_ct_tuple_equal(tuple, &h->tuple) 732 */ 733 static struct nf_conntrack_tuple_hash * 734 ____nf_conntrack_find(struct net *net, const struct nf_conntrack_zone *zone, 735 const struct nf_conntrack_tuple *tuple, u32 hash) 736 { 737 struct nf_conntrack_tuple_hash *h; 738 struct hlist_nulls_head *ct_hash; 739 struct hlist_nulls_node *n; 740 unsigned int bucket, hsize; 741 742 begin: 743 nf_conntrack_get_ht(&ct_hash, &hsize); 744 bucket = reciprocal_scale(hash, hsize); 745 746 hlist_nulls_for_each_entry_rcu(h, n, &ct_hash[bucket], hnnode) { 747 struct nf_conn *ct; 748 749 ct = nf_ct_tuplehash_to_ctrack(h); 750 if (nf_ct_is_expired(ct)) { 751 nf_ct_gc_expired(ct); 752 continue; 753 } 754 755 if (nf_ct_key_equal(h, tuple, zone, net)) 756 return h; 757 } 758 /* 759 * if the nulls value we got at the end of this lookup is 760 * not the expected one, we must restart lookup. 761 * We probably met an item that was moved to another chain. 762 */ 763 if (get_nulls_value(n) != bucket) { 764 NF_CT_STAT_INC_ATOMIC(net, search_restart); 765 goto begin; 766 } 767 768 return NULL; 769 } 770 771 /* Find a connection corresponding to a tuple. */ 772 static struct nf_conntrack_tuple_hash * 773 __nf_conntrack_find_get(struct net *net, const struct nf_conntrack_zone *zone, 774 const struct nf_conntrack_tuple *tuple, u32 hash) 775 { 776 struct nf_conntrack_tuple_hash *h; 777 struct nf_conn *ct; 778 779 rcu_read_lock(); 780 781 h = ____nf_conntrack_find(net, zone, tuple, hash); 782 if (h) { 783 /* We have a candidate that matches the tuple we're interested 784 * in, try to obtain a reference and re-check tuple 785 */ 786 ct = nf_ct_tuplehash_to_ctrack(h); 787 if (likely(atomic_inc_not_zero(&ct->ct_general.use))) { 788 if (likely(nf_ct_key_equal(h, tuple, zone, net))) 789 goto found; 790 791 /* TYPESAFE_BY_RCU recycled the candidate */ 792 nf_ct_put(ct); 793 } 794 795 h = NULL; 796 } 797 found: 798 rcu_read_unlock(); 799 800 return h; 801 } 802 803 struct nf_conntrack_tuple_hash * 804 nf_conntrack_find_get(struct net *net, const struct nf_conntrack_zone *zone, 805 const struct nf_conntrack_tuple *tuple) 806 { 807 return __nf_conntrack_find_get(net, zone, tuple, 808 hash_conntrack_raw(tuple, net)); 809 } 810 EXPORT_SYMBOL_GPL(nf_conntrack_find_get); 811 812 static void __nf_conntrack_hash_insert(struct nf_conn *ct, 813 unsigned int hash, 814 unsigned int reply_hash) 815 { 816 hlist_nulls_add_head_rcu(&ct->tuplehash[IP_CT_DIR_ORIGINAL].hnnode, 817 &nf_conntrack_hash[hash]); 818 hlist_nulls_add_head_rcu(&ct->tuplehash[IP_CT_DIR_REPLY].hnnode, 819 &nf_conntrack_hash[reply_hash]); 820 } 821 822 int 823 nf_conntrack_hash_check_insert(struct nf_conn *ct) 824 { 825 const struct nf_conntrack_zone *zone; 826 struct net *net = nf_ct_net(ct); 827 unsigned int hash, reply_hash; 828 struct nf_conntrack_tuple_hash *h; 829 struct hlist_nulls_node *n; 830 unsigned int sequence; 831 832 zone = nf_ct_zone(ct); 833 834 local_bh_disable(); 835 do { 836 sequence = read_seqcount_begin(&nf_conntrack_generation); 837 hash = hash_conntrack(net, 838 &ct->tuplehash[IP_CT_DIR_ORIGINAL].tuple); 839 reply_hash = hash_conntrack(net, 840 &ct->tuplehash[IP_CT_DIR_REPLY].tuple); 841 } while (nf_conntrack_double_lock(net, hash, reply_hash, sequence)); 842 843 /* See if there's one in the list already, including reverse */ 844 hlist_nulls_for_each_entry(h, n, &nf_conntrack_hash[hash], hnnode) 845 if (nf_ct_key_equal(h, &ct->tuplehash[IP_CT_DIR_ORIGINAL].tuple, 846 zone, net)) 847 goto out; 848 849 hlist_nulls_for_each_entry(h, n, &nf_conntrack_hash[reply_hash], hnnode) 850 if (nf_ct_key_equal(h, &ct->tuplehash[IP_CT_DIR_REPLY].tuple, 851 zone, net)) 852 goto out; 853 854 smp_wmb(); 855 /* The caller holds a reference to this object */ 856 atomic_set(&ct->ct_general.use, 2); 857 __nf_conntrack_hash_insert(ct, hash, reply_hash); 858 nf_conntrack_double_unlock(hash, reply_hash); 859 NF_CT_STAT_INC(net, insert); 860 local_bh_enable(); 861 return 0; 862 863 out: 864 nf_conntrack_double_unlock(hash, reply_hash); 865 local_bh_enable(); 866 return -EEXIST; 867 } 868 EXPORT_SYMBOL_GPL(nf_conntrack_hash_check_insert); 869 870 void nf_ct_acct_add(struct nf_conn *ct, u32 dir, unsigned int packets, 871 unsigned int bytes) 872 { 873 struct nf_conn_acct *acct; 874 875 acct = nf_conn_acct_find(ct); 876 if (acct) { 877 struct nf_conn_counter *counter = acct->counter; 878 879 atomic64_add(packets, &counter[dir].packets); 880 atomic64_add(bytes, &counter[dir].bytes); 881 } 882 } 883 EXPORT_SYMBOL_GPL(nf_ct_acct_add); 884 885 static void nf_ct_acct_merge(struct nf_conn *ct, enum ip_conntrack_info ctinfo, 886 const struct nf_conn *loser_ct) 887 { 888 struct nf_conn_acct *acct; 889 890 acct = nf_conn_acct_find(loser_ct); 891 if (acct) { 892 struct nf_conn_counter *counter = acct->counter; 893 unsigned int bytes; 894 895 /* u32 should be fine since we must have seen one packet. */ 896 bytes = atomic64_read(&counter[CTINFO2DIR(ctinfo)].bytes); 897 nf_ct_acct_update(ct, CTINFO2DIR(ctinfo), bytes); 898 } 899 } 900 901 static void __nf_conntrack_insert_prepare(struct nf_conn *ct) 902 { 903 struct nf_conn_tstamp *tstamp; 904 905 atomic_inc(&ct->ct_general.use); 906 ct->status |= IPS_CONFIRMED; 907 908 /* set conntrack timestamp, if enabled. */ 909 tstamp = nf_conn_tstamp_find(ct); 910 if (tstamp) 911 tstamp->start = ktime_get_real_ns(); 912 } 913 914 /* caller must hold locks to prevent concurrent changes */ 915 static int __nf_ct_resolve_clash(struct sk_buff *skb, 916 struct nf_conntrack_tuple_hash *h) 917 { 918 /* This is the conntrack entry already in hashes that won race. */ 919 struct nf_conn *ct = nf_ct_tuplehash_to_ctrack(h); 920 enum ip_conntrack_info ctinfo; 921 struct nf_conn *loser_ct; 922 923 loser_ct = nf_ct_get(skb, &ctinfo); 924 925 if (nf_ct_is_dying(ct)) 926 return NF_DROP; 927 928 if (((ct->status & IPS_NAT_DONE_MASK) == 0) || 929 nf_ct_match(ct, loser_ct)) { 930 struct net *net = nf_ct_net(ct); 931 932 nf_conntrack_get(&ct->ct_general); 933 934 nf_ct_acct_merge(ct, ctinfo, loser_ct); 935 nf_ct_add_to_dying_list(loser_ct); 936 nf_conntrack_put(&loser_ct->ct_general); 937 nf_ct_set(skb, ct, ctinfo); 938 939 NF_CT_STAT_INC(net, clash_resolve); 940 return NF_ACCEPT; 941 } 942 943 return NF_DROP; 944 } 945 946 /** 947 * nf_ct_resolve_clash_harder - attempt to insert clashing conntrack entry 948 * 949 * @skb: skb that causes the collision 950 * @repl_idx: hash slot for reply direction 951 * 952 * Called when origin or reply direction had a clash. 953 * The skb can be handled without packet drop provided the reply direction 954 * is unique or there the existing entry has the identical tuple in both 955 * directions. 956 * 957 * Caller must hold conntrack table locks to prevent concurrent updates. 958 * 959 * Returns NF_DROP if the clash could not be handled. 960 */ 961 static int nf_ct_resolve_clash_harder(struct sk_buff *skb, u32 repl_idx) 962 { 963 struct nf_conn *loser_ct = (struct nf_conn *)skb_nfct(skb); 964 const struct nf_conntrack_zone *zone; 965 struct nf_conntrack_tuple_hash *h; 966 struct hlist_nulls_node *n; 967 struct net *net; 968 969 zone = nf_ct_zone(loser_ct); 970 net = nf_ct_net(loser_ct); 971 972 /* Reply direction must never result in a clash, unless both origin 973 * and reply tuples are identical. 974 */ 975 hlist_nulls_for_each_entry(h, n, &nf_conntrack_hash[repl_idx], hnnode) { 976 if (nf_ct_key_equal(h, 977 &loser_ct->tuplehash[IP_CT_DIR_REPLY].tuple, 978 zone, net)) 979 return __nf_ct_resolve_clash(skb, h); 980 } 981 982 /* We want the clashing entry to go away real soon: 1 second timeout. */ 983 loser_ct->timeout = nfct_time_stamp + HZ; 984 985 /* IPS_NAT_CLASH removes the entry automatically on the first 986 * reply. Also prevents UDP tracker from moving the entry to 987 * ASSURED state, i.e. the entry can always be evicted under 988 * pressure. 989 */ 990 loser_ct->status |= IPS_FIXED_TIMEOUT | IPS_NAT_CLASH; 991 992 __nf_conntrack_insert_prepare(loser_ct); 993 994 /* fake add for ORIGINAL dir: we want lookups to only find the entry 995 * already in the table. This also hides the clashing entry from 996 * ctnetlink iteration, i.e. conntrack -L won't show them. 997 */ 998 hlist_nulls_add_fake(&loser_ct->tuplehash[IP_CT_DIR_ORIGINAL].hnnode); 999 1000 hlist_nulls_add_head_rcu(&loser_ct->tuplehash[IP_CT_DIR_REPLY].hnnode, 1001 &nf_conntrack_hash[repl_idx]); 1002 1003 NF_CT_STAT_INC(net, clash_resolve); 1004 return NF_ACCEPT; 1005 } 1006 1007 /** 1008 * nf_ct_resolve_clash - attempt to handle clash without packet drop 1009 * 1010 * @skb: skb that causes the clash 1011 * @h: tuplehash of the clashing entry already in table 1012 * @reply_hash: hash slot for reply direction 1013 * 1014 * A conntrack entry can be inserted to the connection tracking table 1015 * if there is no existing entry with an identical tuple. 1016 * 1017 * If there is one, @skb (and the assocated, unconfirmed conntrack) has 1018 * to be dropped. In case @skb is retransmitted, next conntrack lookup 1019 * will find the already-existing entry. 1020 * 1021 * The major problem with such packet drop is the extra delay added by 1022 * the packet loss -- it will take some time for a retransmit to occur 1023 * (or the sender to time out when waiting for a reply). 1024 * 1025 * This function attempts to handle the situation without packet drop. 1026 * 1027 * If @skb has no NAT transformation or if the colliding entries are 1028 * exactly the same, only the to-be-confirmed conntrack entry is discarded 1029 * and @skb is associated with the conntrack entry already in the table. 1030 * 1031 * Failing that, the new, unconfirmed conntrack is still added to the table 1032 * provided that the collision only occurs in the ORIGINAL direction. 1033 * The new entry will be added only in the non-clashing REPLY direction, 1034 * so packets in the ORIGINAL direction will continue to match the existing 1035 * entry. The new entry will also have a fixed timeout so it expires -- 1036 * due to the collision, it will only see reply traffic. 1037 * 1038 * Returns NF_DROP if the clash could not be resolved. 1039 */ 1040 static __cold noinline int 1041 nf_ct_resolve_clash(struct sk_buff *skb, struct nf_conntrack_tuple_hash *h, 1042 u32 reply_hash) 1043 { 1044 /* This is the conntrack entry already in hashes that won race. */ 1045 struct nf_conn *ct = nf_ct_tuplehash_to_ctrack(h); 1046 const struct nf_conntrack_l4proto *l4proto; 1047 enum ip_conntrack_info ctinfo; 1048 struct nf_conn *loser_ct; 1049 struct net *net; 1050 int ret; 1051 1052 loser_ct = nf_ct_get(skb, &ctinfo); 1053 net = nf_ct_net(loser_ct); 1054 1055 l4proto = nf_ct_l4proto_find(nf_ct_protonum(ct)); 1056 if (!l4proto->allow_clash) 1057 goto drop; 1058 1059 ret = __nf_ct_resolve_clash(skb, h); 1060 if (ret == NF_ACCEPT) 1061 return ret; 1062 1063 ret = nf_ct_resolve_clash_harder(skb, reply_hash); 1064 if (ret == NF_ACCEPT) 1065 return ret; 1066 1067 drop: 1068 nf_ct_add_to_dying_list(loser_ct); 1069 NF_CT_STAT_INC(net, drop); 1070 NF_CT_STAT_INC(net, insert_failed); 1071 return NF_DROP; 1072 } 1073 1074 /* Confirm a connection given skb; places it in hash table */ 1075 int 1076 __nf_conntrack_confirm(struct sk_buff *skb) 1077 { 1078 const struct nf_conntrack_zone *zone; 1079 unsigned int hash, reply_hash; 1080 struct nf_conntrack_tuple_hash *h; 1081 struct nf_conn *ct; 1082 struct nf_conn_help *help; 1083 struct hlist_nulls_node *n; 1084 enum ip_conntrack_info ctinfo; 1085 struct net *net; 1086 unsigned int sequence; 1087 int ret = NF_DROP; 1088 1089 ct = nf_ct_get(skb, &ctinfo); 1090 net = nf_ct_net(ct); 1091 1092 /* ipt_REJECT uses nf_conntrack_attach to attach related 1093 ICMP/TCP RST packets in other direction. Actual packet 1094 which created connection will be IP_CT_NEW or for an 1095 expected connection, IP_CT_RELATED. */ 1096 if (CTINFO2DIR(ctinfo) != IP_CT_DIR_ORIGINAL) 1097 return NF_ACCEPT; 1098 1099 zone = nf_ct_zone(ct); 1100 local_bh_disable(); 1101 1102 do { 1103 sequence = read_seqcount_begin(&nf_conntrack_generation); 1104 /* reuse the hash saved before */ 1105 hash = *(unsigned long *)&ct->tuplehash[IP_CT_DIR_REPLY].hnnode.pprev; 1106 hash = scale_hash(hash); 1107 reply_hash = hash_conntrack(net, 1108 &ct->tuplehash[IP_CT_DIR_REPLY].tuple); 1109 1110 } while (nf_conntrack_double_lock(net, hash, reply_hash, sequence)); 1111 1112 /* We're not in hash table, and we refuse to set up related 1113 * connections for unconfirmed conns. But packet copies and 1114 * REJECT will give spurious warnings here. 1115 */ 1116 1117 /* Another skb with the same unconfirmed conntrack may 1118 * win the race. This may happen for bridge(br_flood) 1119 * or broadcast/multicast packets do skb_clone with 1120 * unconfirmed conntrack. 1121 */ 1122 if (unlikely(nf_ct_is_confirmed(ct))) { 1123 WARN_ON_ONCE(1); 1124 nf_conntrack_double_unlock(hash, reply_hash); 1125 local_bh_enable(); 1126 return NF_DROP; 1127 } 1128 1129 pr_debug("Confirming conntrack %p\n", ct); 1130 /* We have to check the DYING flag after unlink to prevent 1131 * a race against nf_ct_get_next_corpse() possibly called from 1132 * user context, else we insert an already 'dead' hash, blocking 1133 * further use of that particular connection -JM. 1134 */ 1135 nf_ct_del_from_dying_or_unconfirmed_list(ct); 1136 1137 if (unlikely(nf_ct_is_dying(ct))) { 1138 nf_ct_add_to_dying_list(ct); 1139 NF_CT_STAT_INC(net, insert_failed); 1140 goto dying; 1141 } 1142 1143 /* See if there's one in the list already, including reverse: 1144 NAT could have grabbed it without realizing, since we're 1145 not in the hash. If there is, we lost race. */ 1146 hlist_nulls_for_each_entry(h, n, &nf_conntrack_hash[hash], hnnode) 1147 if (nf_ct_key_equal(h, &ct->tuplehash[IP_CT_DIR_ORIGINAL].tuple, 1148 zone, net)) 1149 goto out; 1150 1151 hlist_nulls_for_each_entry(h, n, &nf_conntrack_hash[reply_hash], hnnode) 1152 if (nf_ct_key_equal(h, &ct->tuplehash[IP_CT_DIR_REPLY].tuple, 1153 zone, net)) 1154 goto out; 1155 1156 /* Timer relative to confirmation time, not original 1157 setting time, otherwise we'd get timer wrap in 1158 weird delay cases. */ 1159 ct->timeout += nfct_time_stamp; 1160 1161 __nf_conntrack_insert_prepare(ct); 1162 1163 /* Since the lookup is lockless, hash insertion must be done after 1164 * starting the timer and setting the CONFIRMED bit. The RCU barriers 1165 * guarantee that no other CPU can find the conntrack before the above 1166 * stores are visible. 1167 */ 1168 __nf_conntrack_hash_insert(ct, hash, reply_hash); 1169 nf_conntrack_double_unlock(hash, reply_hash); 1170 local_bh_enable(); 1171 1172 help = nfct_help(ct); 1173 if (help && help->helper) 1174 nf_conntrack_event_cache(IPCT_HELPER, ct); 1175 1176 nf_conntrack_event_cache(master_ct(ct) ? 1177 IPCT_RELATED : IPCT_NEW, ct); 1178 return NF_ACCEPT; 1179 1180 out: 1181 ret = nf_ct_resolve_clash(skb, h, reply_hash); 1182 dying: 1183 nf_conntrack_double_unlock(hash, reply_hash); 1184 local_bh_enable(); 1185 return ret; 1186 } 1187 EXPORT_SYMBOL_GPL(__nf_conntrack_confirm); 1188 1189 /* Returns true if a connection correspondings to the tuple (required 1190 for NAT). */ 1191 int 1192 nf_conntrack_tuple_taken(const struct nf_conntrack_tuple *tuple, 1193 const struct nf_conn *ignored_conntrack) 1194 { 1195 struct net *net = nf_ct_net(ignored_conntrack); 1196 const struct nf_conntrack_zone *zone; 1197 struct nf_conntrack_tuple_hash *h; 1198 struct hlist_nulls_head *ct_hash; 1199 unsigned int hash, hsize; 1200 struct hlist_nulls_node *n; 1201 struct nf_conn *ct; 1202 1203 zone = nf_ct_zone(ignored_conntrack); 1204 1205 rcu_read_lock(); 1206 begin: 1207 nf_conntrack_get_ht(&ct_hash, &hsize); 1208 hash = __hash_conntrack(net, tuple, hsize); 1209 1210 hlist_nulls_for_each_entry_rcu(h, n, &ct_hash[hash], hnnode) { 1211 ct = nf_ct_tuplehash_to_ctrack(h); 1212 1213 if (ct == ignored_conntrack) 1214 continue; 1215 1216 if (nf_ct_is_expired(ct)) { 1217 nf_ct_gc_expired(ct); 1218 continue; 1219 } 1220 1221 if (nf_ct_key_equal(h, tuple, zone, net)) { 1222 /* Tuple is taken already, so caller will need to find 1223 * a new source port to use. 1224 * 1225 * Only exception: 1226 * If the *original tuples* are identical, then both 1227 * conntracks refer to the same flow. 1228 * This is a rare situation, it can occur e.g. when 1229 * more than one UDP packet is sent from same socket 1230 * in different threads. 1231 * 1232 * Let nf_ct_resolve_clash() deal with this later. 1233 */ 1234 if (nf_ct_tuple_equal(&ignored_conntrack->tuplehash[IP_CT_DIR_ORIGINAL].tuple, 1235 &ct->tuplehash[IP_CT_DIR_ORIGINAL].tuple) && 1236 nf_ct_zone_equal(ct, zone, IP_CT_DIR_ORIGINAL)) 1237 continue; 1238 1239 NF_CT_STAT_INC_ATOMIC(net, found); 1240 rcu_read_unlock(); 1241 return 1; 1242 } 1243 } 1244 1245 if (get_nulls_value(n) != hash) { 1246 NF_CT_STAT_INC_ATOMIC(net, search_restart); 1247 goto begin; 1248 } 1249 1250 rcu_read_unlock(); 1251 1252 return 0; 1253 } 1254 EXPORT_SYMBOL_GPL(nf_conntrack_tuple_taken); 1255 1256 #define NF_CT_EVICTION_RANGE 8 1257 1258 /* There's a small race here where we may free a just-assured 1259 connection. Too bad: we're in trouble anyway. */ 1260 static unsigned int early_drop_list(struct net *net, 1261 struct hlist_nulls_head *head) 1262 { 1263 struct nf_conntrack_tuple_hash *h; 1264 struct hlist_nulls_node *n; 1265 unsigned int drops = 0; 1266 struct nf_conn *tmp; 1267 1268 hlist_nulls_for_each_entry_rcu(h, n, head, hnnode) { 1269 tmp = nf_ct_tuplehash_to_ctrack(h); 1270 1271 if (test_bit(IPS_OFFLOAD_BIT, &tmp->status)) 1272 continue; 1273 1274 if (nf_ct_is_expired(tmp)) { 1275 nf_ct_gc_expired(tmp); 1276 continue; 1277 } 1278 1279 if (test_bit(IPS_ASSURED_BIT, &tmp->status) || 1280 !net_eq(nf_ct_net(tmp), net) || 1281 nf_ct_is_dying(tmp)) 1282 continue; 1283 1284 if (!atomic_inc_not_zero(&tmp->ct_general.use)) 1285 continue; 1286 1287 /* kill only if still in same netns -- might have moved due to 1288 * SLAB_TYPESAFE_BY_RCU rules. 1289 * 1290 * We steal the timer reference. If that fails timer has 1291 * already fired or someone else deleted it. Just drop ref 1292 * and move to next entry. 1293 */ 1294 if (net_eq(nf_ct_net(tmp), net) && 1295 nf_ct_is_confirmed(tmp) && 1296 nf_ct_delete(tmp, 0, 0)) 1297 drops++; 1298 1299 nf_ct_put(tmp); 1300 } 1301 1302 return drops; 1303 } 1304 1305 static noinline int early_drop(struct net *net, unsigned int hash) 1306 { 1307 unsigned int i, bucket; 1308 1309 for (i = 0; i < NF_CT_EVICTION_RANGE; i++) { 1310 struct hlist_nulls_head *ct_hash; 1311 unsigned int hsize, drops; 1312 1313 rcu_read_lock(); 1314 nf_conntrack_get_ht(&ct_hash, &hsize); 1315 if (!i) 1316 bucket = reciprocal_scale(hash, hsize); 1317 else 1318 bucket = (bucket + 1) % hsize; 1319 1320 drops = early_drop_list(net, &ct_hash[bucket]); 1321 rcu_read_unlock(); 1322 1323 if (drops) { 1324 NF_CT_STAT_ADD_ATOMIC(net, early_drop, drops); 1325 return true; 1326 } 1327 } 1328 1329 return false; 1330 } 1331 1332 static bool gc_worker_skip_ct(const struct nf_conn *ct) 1333 { 1334 return !nf_ct_is_confirmed(ct) || nf_ct_is_dying(ct); 1335 } 1336 1337 static bool gc_worker_can_early_drop(const struct nf_conn *ct) 1338 { 1339 const struct nf_conntrack_l4proto *l4proto; 1340 1341 if (!test_bit(IPS_ASSURED_BIT, &ct->status)) 1342 return true; 1343 1344 l4proto = nf_ct_l4proto_find(nf_ct_protonum(ct)); 1345 if (l4proto->can_early_drop && l4proto->can_early_drop(ct)) 1346 return true; 1347 1348 return false; 1349 } 1350 1351 static void gc_worker(struct work_struct *work) 1352 { 1353 unsigned int min_interval = max(HZ / GC_MAX_BUCKETS_DIV, 1u); 1354 unsigned int i, goal, buckets = 0, expired_count = 0; 1355 unsigned int nf_conntrack_max95 = 0; 1356 struct conntrack_gc_work *gc_work; 1357 unsigned int ratio, scanned = 0; 1358 unsigned long next_run; 1359 1360 gc_work = container_of(work, struct conntrack_gc_work, dwork.work); 1361 1362 goal = nf_conntrack_htable_size / GC_MAX_BUCKETS_DIV; 1363 i = gc_work->last_bucket; 1364 if (gc_work->early_drop) 1365 nf_conntrack_max95 = nf_conntrack_max / 100u * 95u; 1366 1367 do { 1368 struct nf_conntrack_tuple_hash *h; 1369 struct hlist_nulls_head *ct_hash; 1370 struct hlist_nulls_node *n; 1371 unsigned int hashsz; 1372 struct nf_conn *tmp; 1373 1374 i++; 1375 rcu_read_lock(); 1376 1377 nf_conntrack_get_ht(&ct_hash, &hashsz); 1378 if (i >= hashsz) 1379 i = 0; 1380 1381 hlist_nulls_for_each_entry_rcu(h, n, &ct_hash[i], hnnode) { 1382 struct nf_conntrack_net *cnet; 1383 struct net *net; 1384 1385 tmp = nf_ct_tuplehash_to_ctrack(h); 1386 1387 scanned++; 1388 if (test_bit(IPS_OFFLOAD_BIT, &tmp->status)) { 1389 nf_ct_offload_timeout(tmp); 1390 continue; 1391 } 1392 1393 if (nf_ct_is_expired(tmp)) { 1394 nf_ct_gc_expired(tmp); 1395 expired_count++; 1396 continue; 1397 } 1398 1399 if (nf_conntrack_max95 == 0 || gc_worker_skip_ct(tmp)) 1400 continue; 1401 1402 net = nf_ct_net(tmp); 1403 cnet = nf_ct_pernet(net); 1404 if (atomic_read(&cnet->count) < nf_conntrack_max95) 1405 continue; 1406 1407 /* need to take reference to avoid possible races */ 1408 if (!atomic_inc_not_zero(&tmp->ct_general.use)) 1409 continue; 1410 1411 if (gc_worker_skip_ct(tmp)) { 1412 nf_ct_put(tmp); 1413 continue; 1414 } 1415 1416 if (gc_worker_can_early_drop(tmp)) 1417 nf_ct_kill(tmp); 1418 1419 nf_ct_put(tmp); 1420 } 1421 1422 /* could check get_nulls_value() here and restart if ct 1423 * was moved to another chain. But given gc is best-effort 1424 * we will just continue with next hash slot. 1425 */ 1426 rcu_read_unlock(); 1427 cond_resched(); 1428 } while (++buckets < goal); 1429 1430 if (gc_work->exiting) 1431 return; 1432 1433 /* 1434 * Eviction will normally happen from the packet path, and not 1435 * from this gc worker. 1436 * 1437 * This worker is only here to reap expired entries when system went 1438 * idle after a busy period. 1439 * 1440 * The heuristics below are supposed to balance conflicting goals: 1441 * 1442 * 1. Minimize time until we notice a stale entry 1443 * 2. Maximize scan intervals to not waste cycles 1444 * 1445 * Normally, expire ratio will be close to 0. 1446 * 1447 * As soon as a sizeable fraction of the entries have expired 1448 * increase scan frequency. 1449 */ 1450 ratio = scanned ? expired_count * 100 / scanned : 0; 1451 if (ratio > GC_EVICT_RATIO) { 1452 gc_work->next_gc_run = min_interval; 1453 } else { 1454 unsigned int max = GC_MAX_SCAN_JIFFIES / GC_MAX_BUCKETS_DIV; 1455 1456 BUILD_BUG_ON((GC_MAX_SCAN_JIFFIES / GC_MAX_BUCKETS_DIV) == 0); 1457 1458 gc_work->next_gc_run += min_interval; 1459 if (gc_work->next_gc_run > max) 1460 gc_work->next_gc_run = max; 1461 } 1462 1463 next_run = gc_work->next_gc_run; 1464 gc_work->last_bucket = i; 1465 gc_work->early_drop = false; 1466 queue_delayed_work(system_power_efficient_wq, &gc_work->dwork, next_run); 1467 } 1468 1469 static void conntrack_gc_work_init(struct conntrack_gc_work *gc_work) 1470 { 1471 INIT_DEFERRABLE_WORK(&gc_work->dwork, gc_worker); 1472 gc_work->next_gc_run = HZ; 1473 gc_work->exiting = false; 1474 } 1475 1476 static struct nf_conn * 1477 __nf_conntrack_alloc(struct net *net, 1478 const struct nf_conntrack_zone *zone, 1479 const struct nf_conntrack_tuple *orig, 1480 const struct nf_conntrack_tuple *repl, 1481 gfp_t gfp, u32 hash) 1482 { 1483 struct nf_conntrack_net *cnet = nf_ct_pernet(net); 1484 unsigned int ct_count; 1485 struct nf_conn *ct; 1486 1487 /* We don't want any race condition at early drop stage */ 1488 ct_count = atomic_inc_return(&cnet->count); 1489 1490 if (nf_conntrack_max && unlikely(ct_count > nf_conntrack_max)) { 1491 if (!early_drop(net, hash)) { 1492 if (!conntrack_gc_work.early_drop) 1493 conntrack_gc_work.early_drop = true; 1494 atomic_dec(&cnet->count); 1495 net_warn_ratelimited("nf_conntrack: table full, dropping packet\n"); 1496 return ERR_PTR(-ENOMEM); 1497 } 1498 } 1499 1500 /* 1501 * Do not use kmem_cache_zalloc(), as this cache uses 1502 * SLAB_TYPESAFE_BY_RCU. 1503 */ 1504 ct = kmem_cache_alloc(nf_conntrack_cachep, gfp); 1505 if (ct == NULL) 1506 goto out; 1507 1508 spin_lock_init(&ct->lock); 1509 ct->tuplehash[IP_CT_DIR_ORIGINAL].tuple = *orig; 1510 ct->tuplehash[IP_CT_DIR_ORIGINAL].hnnode.pprev = NULL; 1511 ct->tuplehash[IP_CT_DIR_REPLY].tuple = *repl; 1512 /* save hash for reusing when confirming */ 1513 *(unsigned long *)(&ct->tuplehash[IP_CT_DIR_REPLY].hnnode.pprev) = hash; 1514 ct->status = 0; 1515 ct->timeout = 0; 1516 write_pnet(&ct->ct_net, net); 1517 memset(&ct->__nfct_init_offset, 0, 1518 offsetof(struct nf_conn, proto) - 1519 offsetof(struct nf_conn, __nfct_init_offset)); 1520 1521 nf_ct_zone_add(ct, zone); 1522 1523 /* Because we use RCU lookups, we set ct_general.use to zero before 1524 * this is inserted in any list. 1525 */ 1526 atomic_set(&ct->ct_general.use, 0); 1527 return ct; 1528 out: 1529 atomic_dec(&cnet->count); 1530 return ERR_PTR(-ENOMEM); 1531 } 1532 1533 struct nf_conn *nf_conntrack_alloc(struct net *net, 1534 const struct nf_conntrack_zone *zone, 1535 const struct nf_conntrack_tuple *orig, 1536 const struct nf_conntrack_tuple *repl, 1537 gfp_t gfp) 1538 { 1539 return __nf_conntrack_alloc(net, zone, orig, repl, gfp, 0); 1540 } 1541 EXPORT_SYMBOL_GPL(nf_conntrack_alloc); 1542 1543 void nf_conntrack_free(struct nf_conn *ct) 1544 { 1545 struct net *net = nf_ct_net(ct); 1546 struct nf_conntrack_net *cnet; 1547 1548 /* A freed object has refcnt == 0, that's 1549 * the golden rule for SLAB_TYPESAFE_BY_RCU 1550 */ 1551 WARN_ON(atomic_read(&ct->ct_general.use) != 0); 1552 1553 nf_ct_ext_destroy(ct); 1554 kmem_cache_free(nf_conntrack_cachep, ct); 1555 cnet = nf_ct_pernet(net); 1556 1557 smp_mb__before_atomic(); 1558 atomic_dec(&cnet->count); 1559 } 1560 EXPORT_SYMBOL_GPL(nf_conntrack_free); 1561 1562 1563 /* Allocate a new conntrack: we return -ENOMEM if classification 1564 failed due to stress. Otherwise it really is unclassifiable. */ 1565 static noinline struct nf_conntrack_tuple_hash * 1566 init_conntrack(struct net *net, struct nf_conn *tmpl, 1567 const struct nf_conntrack_tuple *tuple, 1568 struct sk_buff *skb, 1569 unsigned int dataoff, u32 hash) 1570 { 1571 struct nf_conn *ct; 1572 struct nf_conn_help *help; 1573 struct nf_conntrack_tuple repl_tuple; 1574 struct nf_conntrack_ecache *ecache; 1575 struct nf_conntrack_expect *exp = NULL; 1576 const struct nf_conntrack_zone *zone; 1577 struct nf_conn_timeout *timeout_ext; 1578 struct nf_conntrack_zone tmp; 1579 struct nf_conntrack_net *cnet; 1580 1581 if (!nf_ct_invert_tuple(&repl_tuple, tuple)) { 1582 pr_debug("Can't invert tuple.\n"); 1583 return NULL; 1584 } 1585 1586 zone = nf_ct_zone_tmpl(tmpl, skb, &tmp); 1587 ct = __nf_conntrack_alloc(net, zone, tuple, &repl_tuple, GFP_ATOMIC, 1588 hash); 1589 if (IS_ERR(ct)) 1590 return (struct nf_conntrack_tuple_hash *)ct; 1591 1592 if (!nf_ct_add_synproxy(ct, tmpl)) { 1593 nf_conntrack_free(ct); 1594 return ERR_PTR(-ENOMEM); 1595 } 1596 1597 timeout_ext = tmpl ? nf_ct_timeout_find(tmpl) : NULL; 1598 1599 if (timeout_ext) 1600 nf_ct_timeout_ext_add(ct, rcu_dereference(timeout_ext->timeout), 1601 GFP_ATOMIC); 1602 1603 nf_ct_acct_ext_add(ct, GFP_ATOMIC); 1604 nf_ct_tstamp_ext_add(ct, GFP_ATOMIC); 1605 nf_ct_labels_ext_add(ct); 1606 1607 ecache = tmpl ? nf_ct_ecache_find(tmpl) : NULL; 1608 nf_ct_ecache_ext_add(ct, ecache ? ecache->ctmask : 0, 1609 ecache ? ecache->expmask : 0, 1610 GFP_ATOMIC); 1611 1612 local_bh_disable(); 1613 cnet = nf_ct_pernet(net); 1614 if (cnet->expect_count) { 1615 spin_lock(&nf_conntrack_expect_lock); 1616 exp = nf_ct_find_expectation(net, zone, tuple); 1617 if (exp) { 1618 pr_debug("expectation arrives ct=%p exp=%p\n", 1619 ct, exp); 1620 /* Welcome, Mr. Bond. We've been expecting you... */ 1621 __set_bit(IPS_EXPECTED_BIT, &ct->status); 1622 /* exp->master safe, refcnt bumped in nf_ct_find_expectation */ 1623 ct->master = exp->master; 1624 if (exp->helper) { 1625 help = nf_ct_helper_ext_add(ct, GFP_ATOMIC); 1626 if (help) 1627 rcu_assign_pointer(help->helper, exp->helper); 1628 } 1629 1630 #ifdef CONFIG_NF_CONNTRACK_MARK 1631 ct->mark = exp->master->mark; 1632 #endif 1633 #ifdef CONFIG_NF_CONNTRACK_SECMARK 1634 ct->secmark = exp->master->secmark; 1635 #endif 1636 NF_CT_STAT_INC(net, expect_new); 1637 } 1638 spin_unlock(&nf_conntrack_expect_lock); 1639 } 1640 if (!exp) 1641 __nf_ct_try_assign_helper(ct, tmpl, GFP_ATOMIC); 1642 1643 /* Now it is inserted into the unconfirmed list, bump refcount */ 1644 nf_conntrack_get(&ct->ct_general); 1645 nf_ct_add_to_unconfirmed_list(ct); 1646 1647 local_bh_enable(); 1648 1649 if (exp) { 1650 if (exp->expectfn) 1651 exp->expectfn(ct, exp); 1652 nf_ct_expect_put(exp); 1653 } 1654 1655 return &ct->tuplehash[IP_CT_DIR_ORIGINAL]; 1656 } 1657 1658 /* On success, returns 0, sets skb->_nfct | ctinfo */ 1659 static int 1660 resolve_normal_ct(struct nf_conn *tmpl, 1661 struct sk_buff *skb, 1662 unsigned int dataoff, 1663 u_int8_t protonum, 1664 const struct nf_hook_state *state) 1665 { 1666 const struct nf_conntrack_zone *zone; 1667 struct nf_conntrack_tuple tuple; 1668 struct nf_conntrack_tuple_hash *h; 1669 enum ip_conntrack_info ctinfo; 1670 struct nf_conntrack_zone tmp; 1671 struct nf_conn *ct; 1672 u32 hash; 1673 1674 if (!nf_ct_get_tuple(skb, skb_network_offset(skb), 1675 dataoff, state->pf, protonum, state->net, 1676 &tuple)) { 1677 pr_debug("Can't get tuple\n"); 1678 return 0; 1679 } 1680 1681 /* look for tuple match */ 1682 zone = nf_ct_zone_tmpl(tmpl, skb, &tmp); 1683 hash = hash_conntrack_raw(&tuple, state->net); 1684 h = __nf_conntrack_find_get(state->net, zone, &tuple, hash); 1685 if (!h) { 1686 h = init_conntrack(state->net, tmpl, &tuple, 1687 skb, dataoff, hash); 1688 if (!h) 1689 return 0; 1690 if (IS_ERR(h)) 1691 return PTR_ERR(h); 1692 } 1693 ct = nf_ct_tuplehash_to_ctrack(h); 1694 1695 /* It exists; we have (non-exclusive) reference. */ 1696 if (NF_CT_DIRECTION(h) == IP_CT_DIR_REPLY) { 1697 ctinfo = IP_CT_ESTABLISHED_REPLY; 1698 } else { 1699 /* Once we've had two way comms, always ESTABLISHED. */ 1700 if (test_bit(IPS_SEEN_REPLY_BIT, &ct->status)) { 1701 pr_debug("normal packet for %p\n", ct); 1702 ctinfo = IP_CT_ESTABLISHED; 1703 } else if (test_bit(IPS_EXPECTED_BIT, &ct->status)) { 1704 pr_debug("related packet for %p\n", ct); 1705 ctinfo = IP_CT_RELATED; 1706 } else { 1707 pr_debug("new packet for %p\n", ct); 1708 ctinfo = IP_CT_NEW; 1709 } 1710 } 1711 nf_ct_set(skb, ct, ctinfo); 1712 return 0; 1713 } 1714 1715 /* 1716 * icmp packets need special treatment to handle error messages that are 1717 * related to a connection. 1718 * 1719 * Callers need to check if skb has a conntrack assigned when this 1720 * helper returns; in such case skb belongs to an already known connection. 1721 */ 1722 static unsigned int __cold 1723 nf_conntrack_handle_icmp(struct nf_conn *tmpl, 1724 struct sk_buff *skb, 1725 unsigned int dataoff, 1726 u8 protonum, 1727 const struct nf_hook_state *state) 1728 { 1729 int ret; 1730 1731 if (state->pf == NFPROTO_IPV4 && protonum == IPPROTO_ICMP) 1732 ret = nf_conntrack_icmpv4_error(tmpl, skb, dataoff, state); 1733 #if IS_ENABLED(CONFIG_IPV6) 1734 else if (state->pf == NFPROTO_IPV6 && protonum == IPPROTO_ICMPV6) 1735 ret = nf_conntrack_icmpv6_error(tmpl, skb, dataoff, state); 1736 #endif 1737 else 1738 return NF_ACCEPT; 1739 1740 if (ret <= 0) 1741 NF_CT_STAT_INC_ATOMIC(state->net, error); 1742 1743 return ret; 1744 } 1745 1746 static int generic_packet(struct nf_conn *ct, struct sk_buff *skb, 1747 enum ip_conntrack_info ctinfo) 1748 { 1749 const unsigned int *timeout = nf_ct_timeout_lookup(ct); 1750 1751 if (!timeout) 1752 timeout = &nf_generic_pernet(nf_ct_net(ct))->timeout; 1753 1754 nf_ct_refresh_acct(ct, ctinfo, skb, *timeout); 1755 return NF_ACCEPT; 1756 } 1757 1758 /* Returns verdict for packet, or -1 for invalid. */ 1759 static int nf_conntrack_handle_packet(struct nf_conn *ct, 1760 struct sk_buff *skb, 1761 unsigned int dataoff, 1762 enum ip_conntrack_info ctinfo, 1763 const struct nf_hook_state *state) 1764 { 1765 switch (nf_ct_protonum(ct)) { 1766 case IPPROTO_TCP: 1767 return nf_conntrack_tcp_packet(ct, skb, dataoff, 1768 ctinfo, state); 1769 case IPPROTO_UDP: 1770 return nf_conntrack_udp_packet(ct, skb, dataoff, 1771 ctinfo, state); 1772 case IPPROTO_ICMP: 1773 return nf_conntrack_icmp_packet(ct, skb, ctinfo, state); 1774 #if IS_ENABLED(CONFIG_IPV6) 1775 case IPPROTO_ICMPV6: 1776 return nf_conntrack_icmpv6_packet(ct, skb, ctinfo, state); 1777 #endif 1778 #ifdef CONFIG_NF_CT_PROTO_UDPLITE 1779 case IPPROTO_UDPLITE: 1780 return nf_conntrack_udplite_packet(ct, skb, dataoff, 1781 ctinfo, state); 1782 #endif 1783 #ifdef CONFIG_NF_CT_PROTO_SCTP 1784 case IPPROTO_SCTP: 1785 return nf_conntrack_sctp_packet(ct, skb, dataoff, 1786 ctinfo, state); 1787 #endif 1788 #ifdef CONFIG_NF_CT_PROTO_DCCP 1789 case IPPROTO_DCCP: 1790 return nf_conntrack_dccp_packet(ct, skb, dataoff, 1791 ctinfo, state); 1792 #endif 1793 #ifdef CONFIG_NF_CT_PROTO_GRE 1794 case IPPROTO_GRE: 1795 return nf_conntrack_gre_packet(ct, skb, dataoff, 1796 ctinfo, state); 1797 #endif 1798 } 1799 1800 return generic_packet(ct, skb, ctinfo); 1801 } 1802 1803 unsigned int 1804 nf_conntrack_in(struct sk_buff *skb, const struct nf_hook_state *state) 1805 { 1806 enum ip_conntrack_info ctinfo; 1807 struct nf_conn *ct, *tmpl; 1808 u_int8_t protonum; 1809 int dataoff, ret; 1810 1811 tmpl = nf_ct_get(skb, &ctinfo); 1812 if (tmpl || ctinfo == IP_CT_UNTRACKED) { 1813 /* Previously seen (loopback or untracked)? Ignore. */ 1814 if ((tmpl && !nf_ct_is_template(tmpl)) || 1815 ctinfo == IP_CT_UNTRACKED) 1816 return NF_ACCEPT; 1817 skb->_nfct = 0; 1818 } 1819 1820 /* rcu_read_lock()ed by nf_hook_thresh */ 1821 dataoff = get_l4proto(skb, skb_network_offset(skb), state->pf, &protonum); 1822 if (dataoff <= 0) { 1823 pr_debug("not prepared to track yet or error occurred\n"); 1824 NF_CT_STAT_INC_ATOMIC(state->net, invalid); 1825 ret = NF_ACCEPT; 1826 goto out; 1827 } 1828 1829 if (protonum == IPPROTO_ICMP || protonum == IPPROTO_ICMPV6) { 1830 ret = nf_conntrack_handle_icmp(tmpl, skb, dataoff, 1831 protonum, state); 1832 if (ret <= 0) { 1833 ret = -ret; 1834 goto out; 1835 } 1836 /* ICMP[v6] protocol trackers may assign one conntrack. */ 1837 if (skb->_nfct) 1838 goto out; 1839 } 1840 repeat: 1841 ret = resolve_normal_ct(tmpl, skb, dataoff, 1842 protonum, state); 1843 if (ret < 0) { 1844 /* Too stressed to deal. */ 1845 NF_CT_STAT_INC_ATOMIC(state->net, drop); 1846 ret = NF_DROP; 1847 goto out; 1848 } 1849 1850 ct = nf_ct_get(skb, &ctinfo); 1851 if (!ct) { 1852 /* Not valid part of a connection */ 1853 NF_CT_STAT_INC_ATOMIC(state->net, invalid); 1854 ret = NF_ACCEPT; 1855 goto out; 1856 } 1857 1858 ret = nf_conntrack_handle_packet(ct, skb, dataoff, ctinfo, state); 1859 if (ret <= 0) { 1860 /* Invalid: inverse of the return code tells 1861 * the netfilter core what to do */ 1862 pr_debug("nf_conntrack_in: Can't track with proto module\n"); 1863 nf_conntrack_put(&ct->ct_general); 1864 skb->_nfct = 0; 1865 NF_CT_STAT_INC_ATOMIC(state->net, invalid); 1866 if (ret == -NF_DROP) 1867 NF_CT_STAT_INC_ATOMIC(state->net, drop); 1868 /* Special case: TCP tracker reports an attempt to reopen a 1869 * closed/aborted connection. We have to go back and create a 1870 * fresh conntrack. 1871 */ 1872 if (ret == -NF_REPEAT) 1873 goto repeat; 1874 ret = -ret; 1875 goto out; 1876 } 1877 1878 if (ctinfo == IP_CT_ESTABLISHED_REPLY && 1879 !test_and_set_bit(IPS_SEEN_REPLY_BIT, &ct->status)) 1880 nf_conntrack_event_cache(IPCT_REPLY, ct); 1881 out: 1882 if (tmpl) 1883 nf_ct_put(tmpl); 1884 1885 return ret; 1886 } 1887 EXPORT_SYMBOL_GPL(nf_conntrack_in); 1888 1889 /* Alter reply tuple (maybe alter helper). This is for NAT, and is 1890 implicitly racy: see __nf_conntrack_confirm */ 1891 void nf_conntrack_alter_reply(struct nf_conn *ct, 1892 const struct nf_conntrack_tuple *newreply) 1893 { 1894 struct nf_conn_help *help = nfct_help(ct); 1895 1896 /* Should be unconfirmed, so not in hash table yet */ 1897 WARN_ON(nf_ct_is_confirmed(ct)); 1898 1899 pr_debug("Altering reply tuple of %p to ", ct); 1900 nf_ct_dump_tuple(newreply); 1901 1902 ct->tuplehash[IP_CT_DIR_REPLY].tuple = *newreply; 1903 if (ct->master || (help && !hlist_empty(&help->expectations))) 1904 return; 1905 1906 rcu_read_lock(); 1907 __nf_ct_try_assign_helper(ct, NULL, GFP_ATOMIC); 1908 rcu_read_unlock(); 1909 } 1910 EXPORT_SYMBOL_GPL(nf_conntrack_alter_reply); 1911 1912 /* Refresh conntrack for this many jiffies and do accounting if do_acct is 1 */ 1913 void __nf_ct_refresh_acct(struct nf_conn *ct, 1914 enum ip_conntrack_info ctinfo, 1915 const struct sk_buff *skb, 1916 u32 extra_jiffies, 1917 bool do_acct) 1918 { 1919 /* Only update if this is not a fixed timeout */ 1920 if (test_bit(IPS_FIXED_TIMEOUT_BIT, &ct->status)) 1921 goto acct; 1922 1923 /* If not in hash table, timer will not be active yet */ 1924 if (nf_ct_is_confirmed(ct)) 1925 extra_jiffies += nfct_time_stamp; 1926 1927 if (READ_ONCE(ct->timeout) != extra_jiffies) 1928 WRITE_ONCE(ct->timeout, extra_jiffies); 1929 acct: 1930 if (do_acct) 1931 nf_ct_acct_update(ct, CTINFO2DIR(ctinfo), skb->len); 1932 } 1933 EXPORT_SYMBOL_GPL(__nf_ct_refresh_acct); 1934 1935 bool nf_ct_kill_acct(struct nf_conn *ct, 1936 enum ip_conntrack_info ctinfo, 1937 const struct sk_buff *skb) 1938 { 1939 nf_ct_acct_update(ct, CTINFO2DIR(ctinfo), skb->len); 1940 1941 return nf_ct_delete(ct, 0, 0); 1942 } 1943 EXPORT_SYMBOL_GPL(nf_ct_kill_acct); 1944 1945 #if IS_ENABLED(CONFIG_NF_CT_NETLINK) 1946 1947 #include <linux/netfilter/nfnetlink.h> 1948 #include <linux/netfilter/nfnetlink_conntrack.h> 1949 #include <linux/mutex.h> 1950 1951 /* Generic function for tcp/udp/sctp/dccp and alike. */ 1952 int nf_ct_port_tuple_to_nlattr(struct sk_buff *skb, 1953 const struct nf_conntrack_tuple *tuple) 1954 { 1955 if (nla_put_be16(skb, CTA_PROTO_SRC_PORT, tuple->src.u.tcp.port) || 1956 nla_put_be16(skb, CTA_PROTO_DST_PORT, tuple->dst.u.tcp.port)) 1957 goto nla_put_failure; 1958 return 0; 1959 1960 nla_put_failure: 1961 return -1; 1962 } 1963 EXPORT_SYMBOL_GPL(nf_ct_port_tuple_to_nlattr); 1964 1965 const struct nla_policy nf_ct_port_nla_policy[CTA_PROTO_MAX+1] = { 1966 [CTA_PROTO_SRC_PORT] = { .type = NLA_U16 }, 1967 [CTA_PROTO_DST_PORT] = { .type = NLA_U16 }, 1968 }; 1969 EXPORT_SYMBOL_GPL(nf_ct_port_nla_policy); 1970 1971 int nf_ct_port_nlattr_to_tuple(struct nlattr *tb[], 1972 struct nf_conntrack_tuple *t, 1973 u_int32_t flags) 1974 { 1975 if (flags & CTA_FILTER_FLAG(CTA_PROTO_SRC_PORT)) { 1976 if (!tb[CTA_PROTO_SRC_PORT]) 1977 return -EINVAL; 1978 1979 t->src.u.tcp.port = nla_get_be16(tb[CTA_PROTO_SRC_PORT]); 1980 } 1981 1982 if (flags & CTA_FILTER_FLAG(CTA_PROTO_DST_PORT)) { 1983 if (!tb[CTA_PROTO_DST_PORT]) 1984 return -EINVAL; 1985 1986 t->dst.u.tcp.port = nla_get_be16(tb[CTA_PROTO_DST_PORT]); 1987 } 1988 1989 return 0; 1990 } 1991 EXPORT_SYMBOL_GPL(nf_ct_port_nlattr_to_tuple); 1992 1993 unsigned int nf_ct_port_nlattr_tuple_size(void) 1994 { 1995 static unsigned int size __read_mostly; 1996 1997 if (!size) 1998 size = nla_policy_len(nf_ct_port_nla_policy, CTA_PROTO_MAX + 1); 1999 2000 return size; 2001 } 2002 EXPORT_SYMBOL_GPL(nf_ct_port_nlattr_tuple_size); 2003 #endif 2004 2005 /* Used by ipt_REJECT and ip6t_REJECT. */ 2006 static void nf_conntrack_attach(struct sk_buff *nskb, const struct sk_buff *skb) 2007 { 2008 struct nf_conn *ct; 2009 enum ip_conntrack_info ctinfo; 2010 2011 /* This ICMP is in reverse direction to the packet which caused it */ 2012 ct = nf_ct_get(skb, &ctinfo); 2013 if (CTINFO2DIR(ctinfo) == IP_CT_DIR_ORIGINAL) 2014 ctinfo = IP_CT_RELATED_REPLY; 2015 else 2016 ctinfo = IP_CT_RELATED; 2017 2018 /* Attach to new skbuff, and increment count */ 2019 nf_ct_set(nskb, ct, ctinfo); 2020 nf_conntrack_get(skb_nfct(nskb)); 2021 } 2022 2023 static int __nf_conntrack_update(struct net *net, struct sk_buff *skb, 2024 struct nf_conn *ct, 2025 enum ip_conntrack_info ctinfo) 2026 { 2027 struct nf_conntrack_tuple_hash *h; 2028 struct nf_conntrack_tuple tuple; 2029 struct nf_nat_hook *nat_hook; 2030 unsigned int status; 2031 int dataoff; 2032 u16 l3num; 2033 u8 l4num; 2034 2035 l3num = nf_ct_l3num(ct); 2036 2037 dataoff = get_l4proto(skb, skb_network_offset(skb), l3num, &l4num); 2038 if (dataoff <= 0) 2039 return -1; 2040 2041 if (!nf_ct_get_tuple(skb, skb_network_offset(skb), dataoff, l3num, 2042 l4num, net, &tuple)) 2043 return -1; 2044 2045 if (ct->status & IPS_SRC_NAT) { 2046 memcpy(tuple.src.u3.all, 2047 ct->tuplehash[IP_CT_DIR_ORIGINAL].tuple.src.u3.all, 2048 sizeof(tuple.src.u3.all)); 2049 tuple.src.u.all = 2050 ct->tuplehash[IP_CT_DIR_ORIGINAL].tuple.src.u.all; 2051 } 2052 2053 if (ct->status & IPS_DST_NAT) { 2054 memcpy(tuple.dst.u3.all, 2055 ct->tuplehash[IP_CT_DIR_ORIGINAL].tuple.dst.u3.all, 2056 sizeof(tuple.dst.u3.all)); 2057 tuple.dst.u.all = 2058 ct->tuplehash[IP_CT_DIR_ORIGINAL].tuple.dst.u.all; 2059 } 2060 2061 h = nf_conntrack_find_get(net, nf_ct_zone(ct), &tuple); 2062 if (!h) 2063 return 0; 2064 2065 /* Store status bits of the conntrack that is clashing to re-do NAT 2066 * mangling according to what it has been done already to this packet. 2067 */ 2068 status = ct->status; 2069 2070 nf_ct_put(ct); 2071 ct = nf_ct_tuplehash_to_ctrack(h); 2072 nf_ct_set(skb, ct, ctinfo); 2073 2074 nat_hook = rcu_dereference(nf_nat_hook); 2075 if (!nat_hook) 2076 return 0; 2077 2078 if (status & IPS_SRC_NAT && 2079 nat_hook->manip_pkt(skb, ct, NF_NAT_MANIP_SRC, 2080 IP_CT_DIR_ORIGINAL) == NF_DROP) 2081 return -1; 2082 2083 if (status & IPS_DST_NAT && 2084 nat_hook->manip_pkt(skb, ct, NF_NAT_MANIP_DST, 2085 IP_CT_DIR_ORIGINAL) == NF_DROP) 2086 return -1; 2087 2088 return 0; 2089 } 2090 2091 /* This packet is coming from userspace via nf_queue, complete the packet 2092 * processing after the helper invocation in nf_confirm(). 2093 */ 2094 static int nf_confirm_cthelper(struct sk_buff *skb, struct nf_conn *ct, 2095 enum ip_conntrack_info ctinfo) 2096 { 2097 const struct nf_conntrack_helper *helper; 2098 const struct nf_conn_help *help; 2099 int protoff; 2100 2101 help = nfct_help(ct); 2102 if (!help) 2103 return 0; 2104 2105 helper = rcu_dereference(help->helper); 2106 if (!(helper->flags & NF_CT_HELPER_F_USERSPACE)) 2107 return 0; 2108 2109 switch (nf_ct_l3num(ct)) { 2110 case NFPROTO_IPV4: 2111 protoff = skb_network_offset(skb) + ip_hdrlen(skb); 2112 break; 2113 #if IS_ENABLED(CONFIG_IPV6) 2114 case NFPROTO_IPV6: { 2115 __be16 frag_off; 2116 u8 pnum; 2117 2118 pnum = ipv6_hdr(skb)->nexthdr; 2119 protoff = ipv6_skip_exthdr(skb, sizeof(struct ipv6hdr), &pnum, 2120 &frag_off); 2121 if (protoff < 0 || (frag_off & htons(~0x7)) != 0) 2122 return 0; 2123 break; 2124 } 2125 #endif 2126 default: 2127 return 0; 2128 } 2129 2130 if (test_bit(IPS_SEQ_ADJUST_BIT, &ct->status) && 2131 !nf_is_loopback_packet(skb)) { 2132 if (!nf_ct_seq_adjust(skb, ct, ctinfo, protoff)) { 2133 NF_CT_STAT_INC_ATOMIC(nf_ct_net(ct), drop); 2134 return -1; 2135 } 2136 } 2137 2138 /* We've seen it coming out the other side: confirm it */ 2139 return nf_conntrack_confirm(skb) == NF_DROP ? - 1 : 0; 2140 } 2141 2142 static int nf_conntrack_update(struct net *net, struct sk_buff *skb) 2143 { 2144 enum ip_conntrack_info ctinfo; 2145 struct nf_conn *ct; 2146 int err; 2147 2148 ct = nf_ct_get(skb, &ctinfo); 2149 if (!ct) 2150 return 0; 2151 2152 if (!nf_ct_is_confirmed(ct)) { 2153 err = __nf_conntrack_update(net, skb, ct, ctinfo); 2154 if (err < 0) 2155 return err; 2156 2157 ct = nf_ct_get(skb, &ctinfo); 2158 } 2159 2160 return nf_confirm_cthelper(skb, ct, ctinfo); 2161 } 2162 2163 static bool nf_conntrack_get_tuple_skb(struct nf_conntrack_tuple *dst_tuple, 2164 const struct sk_buff *skb) 2165 { 2166 const struct nf_conntrack_tuple *src_tuple; 2167 const struct nf_conntrack_tuple_hash *hash; 2168 struct nf_conntrack_tuple srctuple; 2169 enum ip_conntrack_info ctinfo; 2170 struct nf_conn *ct; 2171 2172 ct = nf_ct_get(skb, &ctinfo); 2173 if (ct) { 2174 src_tuple = nf_ct_tuple(ct, CTINFO2DIR(ctinfo)); 2175 memcpy(dst_tuple, src_tuple, sizeof(*dst_tuple)); 2176 return true; 2177 } 2178 2179 if (!nf_ct_get_tuplepr(skb, skb_network_offset(skb), 2180 NFPROTO_IPV4, dev_net(skb->dev), 2181 &srctuple)) 2182 return false; 2183 2184 hash = nf_conntrack_find_get(dev_net(skb->dev), 2185 &nf_ct_zone_dflt, 2186 &srctuple); 2187 if (!hash) 2188 return false; 2189 2190 ct = nf_ct_tuplehash_to_ctrack(hash); 2191 src_tuple = nf_ct_tuple(ct, !hash->tuple.dst.dir); 2192 memcpy(dst_tuple, src_tuple, sizeof(*dst_tuple)); 2193 nf_ct_put(ct); 2194 2195 return true; 2196 } 2197 2198 /* Bring out ya dead! */ 2199 static struct nf_conn * 2200 get_next_corpse(int (*iter)(struct nf_conn *i, void *data), 2201 void *data, unsigned int *bucket) 2202 { 2203 struct nf_conntrack_tuple_hash *h; 2204 struct nf_conn *ct; 2205 struct hlist_nulls_node *n; 2206 spinlock_t *lockp; 2207 2208 for (; *bucket < nf_conntrack_htable_size; (*bucket)++) { 2209 lockp = &nf_conntrack_locks[*bucket % CONNTRACK_LOCKS]; 2210 local_bh_disable(); 2211 nf_conntrack_lock(lockp); 2212 if (*bucket < nf_conntrack_htable_size) { 2213 hlist_nulls_for_each_entry(h, n, &nf_conntrack_hash[*bucket], hnnode) { 2214 if (NF_CT_DIRECTION(h) != IP_CT_DIR_REPLY) 2215 continue; 2216 /* All nf_conn objects are added to hash table twice, one 2217 * for original direction tuple, once for the reply tuple. 2218 * 2219 * Exception: In the IPS_NAT_CLASH case, only the reply 2220 * tuple is added (the original tuple already existed for 2221 * a different object). 2222 * 2223 * We only need to call the iterator once for each 2224 * conntrack, so we just use the 'reply' direction 2225 * tuple while iterating. 2226 */ 2227 ct = nf_ct_tuplehash_to_ctrack(h); 2228 if (iter(ct, data)) 2229 goto found; 2230 } 2231 } 2232 spin_unlock(lockp); 2233 local_bh_enable(); 2234 cond_resched(); 2235 } 2236 2237 return NULL; 2238 found: 2239 atomic_inc(&ct->ct_general.use); 2240 spin_unlock(lockp); 2241 local_bh_enable(); 2242 return ct; 2243 } 2244 2245 static void nf_ct_iterate_cleanup(int (*iter)(struct nf_conn *i, void *data), 2246 void *data, u32 portid, int report) 2247 { 2248 unsigned int bucket = 0, sequence; 2249 struct nf_conn *ct; 2250 2251 might_sleep(); 2252 2253 for (;;) { 2254 sequence = read_seqcount_begin(&nf_conntrack_generation); 2255 2256 while ((ct = get_next_corpse(iter, data, &bucket)) != NULL) { 2257 /* Time to push up daises... */ 2258 2259 nf_ct_delete(ct, portid, report); 2260 nf_ct_put(ct); 2261 cond_resched(); 2262 } 2263 2264 if (!read_seqcount_retry(&nf_conntrack_generation, sequence)) 2265 break; 2266 bucket = 0; 2267 } 2268 } 2269 2270 struct iter_data { 2271 int (*iter)(struct nf_conn *i, void *data); 2272 void *data; 2273 struct net *net; 2274 }; 2275 2276 static int iter_net_only(struct nf_conn *i, void *data) 2277 { 2278 struct iter_data *d = data; 2279 2280 if (!net_eq(d->net, nf_ct_net(i))) 2281 return 0; 2282 2283 return d->iter(i, d->data); 2284 } 2285 2286 static void 2287 __nf_ct_unconfirmed_destroy(struct net *net) 2288 { 2289 int cpu; 2290 2291 for_each_possible_cpu(cpu) { 2292 struct nf_conntrack_tuple_hash *h; 2293 struct hlist_nulls_node *n; 2294 struct ct_pcpu *pcpu; 2295 2296 pcpu = per_cpu_ptr(net->ct.pcpu_lists, cpu); 2297 2298 spin_lock_bh(&pcpu->lock); 2299 hlist_nulls_for_each_entry(h, n, &pcpu->unconfirmed, hnnode) { 2300 struct nf_conn *ct; 2301 2302 ct = nf_ct_tuplehash_to_ctrack(h); 2303 2304 /* we cannot call iter() on unconfirmed list, the 2305 * owning cpu can reallocate ct->ext at any time. 2306 */ 2307 set_bit(IPS_DYING_BIT, &ct->status); 2308 } 2309 spin_unlock_bh(&pcpu->lock); 2310 cond_resched(); 2311 } 2312 } 2313 2314 void nf_ct_unconfirmed_destroy(struct net *net) 2315 { 2316 struct nf_conntrack_net *cnet = nf_ct_pernet(net); 2317 2318 might_sleep(); 2319 2320 if (atomic_read(&cnet->count) > 0) { 2321 __nf_ct_unconfirmed_destroy(net); 2322 nf_queue_nf_hook_drop(net); 2323 synchronize_net(); 2324 } 2325 } 2326 EXPORT_SYMBOL_GPL(nf_ct_unconfirmed_destroy); 2327 2328 void nf_ct_iterate_cleanup_net(struct net *net, 2329 int (*iter)(struct nf_conn *i, void *data), 2330 void *data, u32 portid, int report) 2331 { 2332 struct nf_conntrack_net *cnet = nf_ct_pernet(net); 2333 struct iter_data d; 2334 2335 might_sleep(); 2336 2337 if (atomic_read(&cnet->count) == 0) 2338 return; 2339 2340 d.iter = iter; 2341 d.data = data; 2342 d.net = net; 2343 2344 nf_ct_iterate_cleanup(iter_net_only, &d, portid, report); 2345 } 2346 EXPORT_SYMBOL_GPL(nf_ct_iterate_cleanup_net); 2347 2348 /** 2349 * nf_ct_iterate_destroy - destroy unconfirmed conntracks and iterate table 2350 * @iter: callback to invoke for each conntrack 2351 * @data: data to pass to @iter 2352 * 2353 * Like nf_ct_iterate_cleanup, but first marks conntracks on the 2354 * unconfirmed list as dying (so they will not be inserted into 2355 * main table). 2356 * 2357 * Can only be called in module exit path. 2358 */ 2359 void 2360 nf_ct_iterate_destroy(int (*iter)(struct nf_conn *i, void *data), void *data) 2361 { 2362 struct net *net; 2363 2364 down_read(&net_rwsem); 2365 for_each_net(net) { 2366 struct nf_conntrack_net *cnet = nf_ct_pernet(net); 2367 2368 if (atomic_read(&cnet->count) == 0) 2369 continue; 2370 __nf_ct_unconfirmed_destroy(net); 2371 nf_queue_nf_hook_drop(net); 2372 } 2373 up_read(&net_rwsem); 2374 2375 /* Need to wait for netns cleanup worker to finish, if its 2376 * running -- it might have deleted a net namespace from 2377 * the global list, so our __nf_ct_unconfirmed_destroy() might 2378 * not have affected all namespaces. 2379 */ 2380 net_ns_barrier(); 2381 2382 /* a conntrack could have been unlinked from unconfirmed list 2383 * before we grabbed pcpu lock in __nf_ct_unconfirmed_destroy(). 2384 * This makes sure its inserted into conntrack table. 2385 */ 2386 synchronize_net(); 2387 2388 nf_ct_iterate_cleanup(iter, data, 0, 0); 2389 } 2390 EXPORT_SYMBOL_GPL(nf_ct_iterate_destroy); 2391 2392 static int kill_all(struct nf_conn *i, void *data) 2393 { 2394 return net_eq(nf_ct_net(i), data); 2395 } 2396 2397 void nf_conntrack_cleanup_start(void) 2398 { 2399 conntrack_gc_work.exiting = true; 2400 RCU_INIT_POINTER(ip_ct_attach, NULL); 2401 } 2402 2403 void nf_conntrack_cleanup_end(void) 2404 { 2405 RCU_INIT_POINTER(nf_ct_hook, NULL); 2406 cancel_delayed_work_sync(&conntrack_gc_work.dwork); 2407 kvfree(nf_conntrack_hash); 2408 2409 nf_conntrack_proto_fini(); 2410 nf_conntrack_seqadj_fini(); 2411 nf_conntrack_labels_fini(); 2412 nf_conntrack_helper_fini(); 2413 nf_conntrack_timeout_fini(); 2414 nf_conntrack_ecache_fini(); 2415 nf_conntrack_tstamp_fini(); 2416 nf_conntrack_acct_fini(); 2417 nf_conntrack_expect_fini(); 2418 2419 kmem_cache_destroy(nf_conntrack_cachep); 2420 } 2421 2422 /* 2423 * Mishearing the voices in his head, our hero wonders how he's 2424 * supposed to kill the mall. 2425 */ 2426 void nf_conntrack_cleanup_net(struct net *net) 2427 { 2428 LIST_HEAD(single); 2429 2430 list_add(&net->exit_list, &single); 2431 nf_conntrack_cleanup_net_list(&single); 2432 } 2433 2434 void nf_conntrack_cleanup_net_list(struct list_head *net_exit_list) 2435 { 2436 int busy; 2437 struct net *net; 2438 2439 /* 2440 * This makes sure all current packets have passed through 2441 * netfilter framework. Roll on, two-stage module 2442 * delete... 2443 */ 2444 synchronize_net(); 2445 i_see_dead_people: 2446 busy = 0; 2447 list_for_each_entry(net, net_exit_list, exit_list) { 2448 struct nf_conntrack_net *cnet = nf_ct_pernet(net); 2449 2450 nf_ct_iterate_cleanup(kill_all, net, 0, 0); 2451 if (atomic_read(&cnet->count) != 0) 2452 busy = 1; 2453 } 2454 if (busy) { 2455 schedule(); 2456 goto i_see_dead_people; 2457 } 2458 2459 list_for_each_entry(net, net_exit_list, exit_list) { 2460 nf_conntrack_proto_pernet_fini(net); 2461 nf_conntrack_ecache_pernet_fini(net); 2462 nf_conntrack_expect_pernet_fini(net); 2463 free_percpu(net->ct.stat); 2464 free_percpu(net->ct.pcpu_lists); 2465 } 2466 } 2467 2468 void *nf_ct_alloc_hashtable(unsigned int *sizep, int nulls) 2469 { 2470 struct hlist_nulls_head *hash; 2471 unsigned int nr_slots, i; 2472 2473 if (*sizep > (UINT_MAX / sizeof(struct hlist_nulls_head))) 2474 return NULL; 2475 2476 BUILD_BUG_ON(sizeof(struct hlist_nulls_head) != sizeof(struct hlist_head)); 2477 nr_slots = *sizep = roundup(*sizep, PAGE_SIZE / sizeof(struct hlist_nulls_head)); 2478 2479 hash = kvcalloc(nr_slots, sizeof(struct hlist_nulls_head), GFP_KERNEL); 2480 2481 if (hash && nulls) 2482 for (i = 0; i < nr_slots; i++) 2483 INIT_HLIST_NULLS_HEAD(&hash[i], i); 2484 2485 return hash; 2486 } 2487 EXPORT_SYMBOL_GPL(nf_ct_alloc_hashtable); 2488 2489 int nf_conntrack_hash_resize(unsigned int hashsize) 2490 { 2491 int i, bucket; 2492 unsigned int old_size; 2493 struct hlist_nulls_head *hash, *old_hash; 2494 struct nf_conntrack_tuple_hash *h; 2495 struct nf_conn *ct; 2496 2497 if (!hashsize) 2498 return -EINVAL; 2499 2500 hash = nf_ct_alloc_hashtable(&hashsize, 1); 2501 if (!hash) 2502 return -ENOMEM; 2503 2504 old_size = nf_conntrack_htable_size; 2505 if (old_size == hashsize) { 2506 kvfree(hash); 2507 return 0; 2508 } 2509 2510 local_bh_disable(); 2511 nf_conntrack_all_lock(); 2512 write_seqcount_begin(&nf_conntrack_generation); 2513 2514 /* Lookups in the old hash might happen in parallel, which means we 2515 * might get false negatives during connection lookup. New connections 2516 * created because of a false negative won't make it into the hash 2517 * though since that required taking the locks. 2518 */ 2519 2520 for (i = 0; i < nf_conntrack_htable_size; i++) { 2521 while (!hlist_nulls_empty(&nf_conntrack_hash[i])) { 2522 h = hlist_nulls_entry(nf_conntrack_hash[i].first, 2523 struct nf_conntrack_tuple_hash, hnnode); 2524 ct = nf_ct_tuplehash_to_ctrack(h); 2525 hlist_nulls_del_rcu(&h->hnnode); 2526 bucket = __hash_conntrack(nf_ct_net(ct), 2527 &h->tuple, hashsize); 2528 hlist_nulls_add_head_rcu(&h->hnnode, &hash[bucket]); 2529 } 2530 } 2531 old_size = nf_conntrack_htable_size; 2532 old_hash = nf_conntrack_hash; 2533 2534 nf_conntrack_hash = hash; 2535 nf_conntrack_htable_size = hashsize; 2536 2537 write_seqcount_end(&nf_conntrack_generation); 2538 nf_conntrack_all_unlock(); 2539 local_bh_enable(); 2540 2541 synchronize_net(); 2542 kvfree(old_hash); 2543 return 0; 2544 } 2545 2546 int nf_conntrack_set_hashsize(const char *val, const struct kernel_param *kp) 2547 { 2548 unsigned int hashsize; 2549 int rc; 2550 2551 if (current->nsproxy->net_ns != &init_net) 2552 return -EOPNOTSUPP; 2553 2554 /* On boot, we can set this without any fancy locking. */ 2555 if (!nf_conntrack_hash) 2556 return param_set_uint(val, kp); 2557 2558 rc = kstrtouint(val, 0, &hashsize); 2559 if (rc) 2560 return rc; 2561 2562 return nf_conntrack_hash_resize(hashsize); 2563 } 2564 2565 static __always_inline unsigned int total_extension_size(void) 2566 { 2567 /* remember to add new extensions below */ 2568 BUILD_BUG_ON(NF_CT_EXT_NUM > 9); 2569 2570 return sizeof(struct nf_ct_ext) + 2571 sizeof(struct nf_conn_help) 2572 #if IS_ENABLED(CONFIG_NF_NAT) 2573 + sizeof(struct nf_conn_nat) 2574 #endif 2575 + sizeof(struct nf_conn_seqadj) 2576 + sizeof(struct nf_conn_acct) 2577 #ifdef CONFIG_NF_CONNTRACK_EVENTS 2578 + sizeof(struct nf_conntrack_ecache) 2579 #endif 2580 #ifdef CONFIG_NF_CONNTRACK_TIMESTAMP 2581 + sizeof(struct nf_conn_tstamp) 2582 #endif 2583 #ifdef CONFIG_NF_CONNTRACK_TIMEOUT 2584 + sizeof(struct nf_conn_timeout) 2585 #endif 2586 #ifdef CONFIG_NF_CONNTRACK_LABELS 2587 + sizeof(struct nf_conn_labels) 2588 #endif 2589 #if IS_ENABLED(CONFIG_NETFILTER_SYNPROXY) 2590 + sizeof(struct nf_conn_synproxy) 2591 #endif 2592 ; 2593 }; 2594 2595 int nf_conntrack_init_start(void) 2596 { 2597 unsigned long nr_pages = totalram_pages(); 2598 int max_factor = 8; 2599 int ret = -ENOMEM; 2600 int i; 2601 2602 /* struct nf_ct_ext uses u8 to store offsets/size */ 2603 BUILD_BUG_ON(total_extension_size() > 255u); 2604 2605 seqcount_spinlock_init(&nf_conntrack_generation, 2606 &nf_conntrack_locks_all_lock); 2607 2608 for (i = 0; i < CONNTRACK_LOCKS; i++) 2609 spin_lock_init(&nf_conntrack_locks[i]); 2610 2611 if (!nf_conntrack_htable_size) { 2612 /* Idea from tcp.c: use 1/16384 of memory. 2613 * On i386: 32MB machine has 512 buckets. 2614 * >= 1GB machines have 16384 buckets. 2615 * >= 4GB machines have 65536 buckets. 2616 */ 2617 nf_conntrack_htable_size 2618 = (((nr_pages << PAGE_SHIFT) / 16384) 2619 / sizeof(struct hlist_head)); 2620 if (nr_pages > (4 * (1024 * 1024 * 1024 / PAGE_SIZE))) 2621 nf_conntrack_htable_size = 65536; 2622 else if (nr_pages > (1024 * 1024 * 1024 / PAGE_SIZE)) 2623 nf_conntrack_htable_size = 16384; 2624 if (nf_conntrack_htable_size < 32) 2625 nf_conntrack_htable_size = 32; 2626 2627 /* Use a max. factor of four by default to get the same max as 2628 * with the old struct list_heads. When a table size is given 2629 * we use the old value of 8 to avoid reducing the max. 2630 * entries. */ 2631 max_factor = 4; 2632 } 2633 2634 nf_conntrack_hash = nf_ct_alloc_hashtable(&nf_conntrack_htable_size, 1); 2635 if (!nf_conntrack_hash) 2636 return -ENOMEM; 2637 2638 nf_conntrack_max = max_factor * nf_conntrack_htable_size; 2639 2640 nf_conntrack_cachep = kmem_cache_create("nf_conntrack", 2641 sizeof(struct nf_conn), 2642 NFCT_INFOMASK + 1, 2643 SLAB_TYPESAFE_BY_RCU | SLAB_HWCACHE_ALIGN, NULL); 2644 if (!nf_conntrack_cachep) 2645 goto err_cachep; 2646 2647 ret = nf_conntrack_expect_init(); 2648 if (ret < 0) 2649 goto err_expect; 2650 2651 ret = nf_conntrack_acct_init(); 2652 if (ret < 0) 2653 goto err_acct; 2654 2655 ret = nf_conntrack_tstamp_init(); 2656 if (ret < 0) 2657 goto err_tstamp; 2658 2659 ret = nf_conntrack_ecache_init(); 2660 if (ret < 0) 2661 goto err_ecache; 2662 2663 ret = nf_conntrack_timeout_init(); 2664 if (ret < 0) 2665 goto err_timeout; 2666 2667 ret = nf_conntrack_helper_init(); 2668 if (ret < 0) 2669 goto err_helper; 2670 2671 ret = nf_conntrack_labels_init(); 2672 if (ret < 0) 2673 goto err_labels; 2674 2675 ret = nf_conntrack_seqadj_init(); 2676 if (ret < 0) 2677 goto err_seqadj; 2678 2679 ret = nf_conntrack_proto_init(); 2680 if (ret < 0) 2681 goto err_proto; 2682 2683 conntrack_gc_work_init(&conntrack_gc_work); 2684 queue_delayed_work(system_power_efficient_wq, &conntrack_gc_work.dwork, HZ); 2685 2686 return 0; 2687 2688 err_proto: 2689 nf_conntrack_seqadj_fini(); 2690 err_seqadj: 2691 nf_conntrack_labels_fini(); 2692 err_labels: 2693 nf_conntrack_helper_fini(); 2694 err_helper: 2695 nf_conntrack_timeout_fini(); 2696 err_timeout: 2697 nf_conntrack_ecache_fini(); 2698 err_ecache: 2699 nf_conntrack_tstamp_fini(); 2700 err_tstamp: 2701 nf_conntrack_acct_fini(); 2702 err_acct: 2703 nf_conntrack_expect_fini(); 2704 err_expect: 2705 kmem_cache_destroy(nf_conntrack_cachep); 2706 err_cachep: 2707 kvfree(nf_conntrack_hash); 2708 return ret; 2709 } 2710 2711 static struct nf_ct_hook nf_conntrack_hook = { 2712 .update = nf_conntrack_update, 2713 .destroy = destroy_conntrack, 2714 .get_tuple_skb = nf_conntrack_get_tuple_skb, 2715 }; 2716 2717 void nf_conntrack_init_end(void) 2718 { 2719 /* For use by REJECT target */ 2720 RCU_INIT_POINTER(ip_ct_attach, nf_conntrack_attach); 2721 RCU_INIT_POINTER(nf_ct_hook, &nf_conntrack_hook); 2722 } 2723 2724 /* 2725 * We need to use special "null" values, not used in hash table 2726 */ 2727 #define UNCONFIRMED_NULLS_VAL ((1<<30)+0) 2728 #define DYING_NULLS_VAL ((1<<30)+1) 2729 2730 int nf_conntrack_init_net(struct net *net) 2731 { 2732 struct nf_conntrack_net *cnet = nf_ct_pernet(net); 2733 int ret = -ENOMEM; 2734 int cpu; 2735 2736 BUILD_BUG_ON(IP_CT_UNTRACKED == IP_CT_NUMBER); 2737 BUILD_BUG_ON_NOT_POWER_OF_2(CONNTRACK_LOCKS); 2738 atomic_set(&cnet->count, 0); 2739 2740 net->ct.pcpu_lists = alloc_percpu(struct ct_pcpu); 2741 if (!net->ct.pcpu_lists) 2742 goto err_stat; 2743 2744 for_each_possible_cpu(cpu) { 2745 struct ct_pcpu *pcpu = per_cpu_ptr(net->ct.pcpu_lists, cpu); 2746 2747 spin_lock_init(&pcpu->lock); 2748 INIT_HLIST_NULLS_HEAD(&pcpu->unconfirmed, UNCONFIRMED_NULLS_VAL); 2749 INIT_HLIST_NULLS_HEAD(&pcpu->dying, DYING_NULLS_VAL); 2750 } 2751 2752 net->ct.stat = alloc_percpu(struct ip_conntrack_stat); 2753 if (!net->ct.stat) 2754 goto err_pcpu_lists; 2755 2756 ret = nf_conntrack_expect_pernet_init(net); 2757 if (ret < 0) 2758 goto err_expect; 2759 2760 nf_conntrack_acct_pernet_init(net); 2761 nf_conntrack_tstamp_pernet_init(net); 2762 nf_conntrack_ecache_pernet_init(net); 2763 nf_conntrack_helper_pernet_init(net); 2764 nf_conntrack_proto_pernet_init(net); 2765 2766 return 0; 2767 2768 err_expect: 2769 free_percpu(net->ct.stat); 2770 err_pcpu_lists: 2771 free_percpu(net->ct.pcpu_lists); 2772 err_stat: 2773 return ret; 2774 } 2775