1 /* Connection state tracking for netfilter. This is separated from, 2 but required by, the NAT layer; it can also be used by an iptables 3 extension. */ 4 5 /* (C) 1999-2001 Paul `Rusty' Russell 6 * (C) 2002-2006 Netfilter Core Team <coreteam@netfilter.org> 7 * (C) 2003,2004 USAGI/WIDE Project <http://www.linux-ipv6.org> 8 * (C) 2005-2012 Patrick McHardy <kaber@trash.net> 9 * 10 * This program is free software; you can redistribute it and/or modify 11 * it under the terms of the GNU General Public License version 2 as 12 * published by the Free Software Foundation. 13 */ 14 15 #include <linux/types.h> 16 #include <linux/netfilter.h> 17 #include <linux/module.h> 18 #include <linux/sched.h> 19 #include <linux/skbuff.h> 20 #include <linux/proc_fs.h> 21 #include <linux/vmalloc.h> 22 #include <linux/stddef.h> 23 #include <linux/slab.h> 24 #include <linux/random.h> 25 #include <linux/jhash.h> 26 #include <linux/err.h> 27 #include <linux/percpu.h> 28 #include <linux/moduleparam.h> 29 #include <linux/notifier.h> 30 #include <linux/kernel.h> 31 #include <linux/netdevice.h> 32 #include <linux/socket.h> 33 #include <linux/mm.h> 34 #include <linux/nsproxy.h> 35 #include <linux/rculist_nulls.h> 36 37 #include <net/netfilter/nf_conntrack.h> 38 #include <net/netfilter/nf_conntrack_l3proto.h> 39 #include <net/netfilter/nf_conntrack_l4proto.h> 40 #include <net/netfilter/nf_conntrack_expect.h> 41 #include <net/netfilter/nf_conntrack_helper.h> 42 #include <net/netfilter/nf_conntrack_seqadj.h> 43 #include <net/netfilter/nf_conntrack_core.h> 44 #include <net/netfilter/nf_conntrack_extend.h> 45 #include <net/netfilter/nf_conntrack_acct.h> 46 #include <net/netfilter/nf_conntrack_ecache.h> 47 #include <net/netfilter/nf_conntrack_zones.h> 48 #include <net/netfilter/nf_conntrack_timestamp.h> 49 #include <net/netfilter/nf_conntrack_timeout.h> 50 #include <net/netfilter/nf_conntrack_labels.h> 51 #include <net/netfilter/nf_conntrack_synproxy.h> 52 #include <net/netfilter/nf_nat.h> 53 #include <net/netfilter/nf_nat_core.h> 54 #include <net/netfilter/nf_nat_helper.h> 55 56 #define NF_CONNTRACK_VERSION "0.5.0" 57 58 int (*nfnetlink_parse_nat_setup_hook)(struct nf_conn *ct, 59 enum nf_nat_manip_type manip, 60 const struct nlattr *attr) __read_mostly; 61 EXPORT_SYMBOL_GPL(nfnetlink_parse_nat_setup_hook); 62 63 __cacheline_aligned_in_smp spinlock_t nf_conntrack_locks[CONNTRACK_LOCKS]; 64 EXPORT_SYMBOL_GPL(nf_conntrack_locks); 65 66 __cacheline_aligned_in_smp DEFINE_SPINLOCK(nf_conntrack_expect_lock); 67 EXPORT_SYMBOL_GPL(nf_conntrack_expect_lock); 68 69 static void nf_conntrack_double_unlock(unsigned int h1, unsigned int h2) 70 { 71 h1 %= CONNTRACK_LOCKS; 72 h2 %= CONNTRACK_LOCKS; 73 spin_unlock(&nf_conntrack_locks[h1]); 74 if (h1 != h2) 75 spin_unlock(&nf_conntrack_locks[h2]); 76 } 77 78 /* return true if we need to recompute hashes (in case hash table was resized) */ 79 static bool nf_conntrack_double_lock(struct net *net, unsigned int h1, 80 unsigned int h2, unsigned int sequence) 81 { 82 h1 %= CONNTRACK_LOCKS; 83 h2 %= CONNTRACK_LOCKS; 84 if (h1 <= h2) { 85 spin_lock(&nf_conntrack_locks[h1]); 86 if (h1 != h2) 87 spin_lock_nested(&nf_conntrack_locks[h2], 88 SINGLE_DEPTH_NESTING); 89 } else { 90 spin_lock(&nf_conntrack_locks[h2]); 91 spin_lock_nested(&nf_conntrack_locks[h1], 92 SINGLE_DEPTH_NESTING); 93 } 94 if (read_seqcount_retry(&net->ct.generation, sequence)) { 95 nf_conntrack_double_unlock(h1, h2); 96 return true; 97 } 98 return false; 99 } 100 101 static void nf_conntrack_all_lock(void) 102 { 103 int i; 104 105 for (i = 0; i < CONNTRACK_LOCKS; i++) 106 spin_lock_nested(&nf_conntrack_locks[i], i); 107 } 108 109 static void nf_conntrack_all_unlock(void) 110 { 111 int i; 112 113 for (i = 0; i < CONNTRACK_LOCKS; i++) 114 spin_unlock(&nf_conntrack_locks[i]); 115 } 116 117 unsigned int nf_conntrack_htable_size __read_mostly; 118 EXPORT_SYMBOL_GPL(nf_conntrack_htable_size); 119 120 unsigned int nf_conntrack_max __read_mostly; 121 EXPORT_SYMBOL_GPL(nf_conntrack_max); 122 123 DEFINE_PER_CPU(struct nf_conn, nf_conntrack_untracked); 124 EXPORT_PER_CPU_SYMBOL(nf_conntrack_untracked); 125 126 unsigned int nf_conntrack_hash_rnd __read_mostly; 127 EXPORT_SYMBOL_GPL(nf_conntrack_hash_rnd); 128 129 static u32 hash_conntrack_raw(const struct nf_conntrack_tuple *tuple) 130 { 131 unsigned int n; 132 133 /* The direction must be ignored, so we hash everything up to the 134 * destination ports (which is a multiple of 4) and treat the last 135 * three bytes manually. 136 */ 137 n = (sizeof(tuple->src) + sizeof(tuple->dst.u3)) / sizeof(u32); 138 return jhash2((u32 *)tuple, n, nf_conntrack_hash_rnd ^ 139 (((__force __u16)tuple->dst.u.all << 16) | 140 tuple->dst.protonum)); 141 } 142 143 static u32 __hash_bucket(u32 hash, unsigned int size) 144 { 145 return reciprocal_scale(hash, size); 146 } 147 148 static u32 hash_bucket(u32 hash, const struct net *net) 149 { 150 return __hash_bucket(hash, net->ct.htable_size); 151 } 152 153 static u_int32_t __hash_conntrack(const struct nf_conntrack_tuple *tuple, 154 unsigned int size) 155 { 156 return __hash_bucket(hash_conntrack_raw(tuple), size); 157 } 158 159 static inline u_int32_t hash_conntrack(const struct net *net, 160 const struct nf_conntrack_tuple *tuple) 161 { 162 return __hash_conntrack(tuple, net->ct.htable_size); 163 } 164 165 bool 166 nf_ct_get_tuple(const struct sk_buff *skb, 167 unsigned int nhoff, 168 unsigned int dataoff, 169 u_int16_t l3num, 170 u_int8_t protonum, 171 struct net *net, 172 struct nf_conntrack_tuple *tuple, 173 const struct nf_conntrack_l3proto *l3proto, 174 const struct nf_conntrack_l4proto *l4proto) 175 { 176 memset(tuple, 0, sizeof(*tuple)); 177 178 tuple->src.l3num = l3num; 179 if (l3proto->pkt_to_tuple(skb, nhoff, tuple) == 0) 180 return false; 181 182 tuple->dst.protonum = protonum; 183 tuple->dst.dir = IP_CT_DIR_ORIGINAL; 184 185 return l4proto->pkt_to_tuple(skb, dataoff, net, tuple); 186 } 187 EXPORT_SYMBOL_GPL(nf_ct_get_tuple); 188 189 bool nf_ct_get_tuplepr(const struct sk_buff *skb, unsigned int nhoff, 190 u_int16_t l3num, 191 struct net *net, struct nf_conntrack_tuple *tuple) 192 { 193 struct nf_conntrack_l3proto *l3proto; 194 struct nf_conntrack_l4proto *l4proto; 195 unsigned int protoff; 196 u_int8_t protonum; 197 int ret; 198 199 rcu_read_lock(); 200 201 l3proto = __nf_ct_l3proto_find(l3num); 202 ret = l3proto->get_l4proto(skb, nhoff, &protoff, &protonum); 203 if (ret != NF_ACCEPT) { 204 rcu_read_unlock(); 205 return false; 206 } 207 208 l4proto = __nf_ct_l4proto_find(l3num, protonum); 209 210 ret = nf_ct_get_tuple(skb, nhoff, protoff, l3num, protonum, net, tuple, 211 l3proto, l4proto); 212 213 rcu_read_unlock(); 214 return ret; 215 } 216 EXPORT_SYMBOL_GPL(nf_ct_get_tuplepr); 217 218 bool 219 nf_ct_invert_tuple(struct nf_conntrack_tuple *inverse, 220 const struct nf_conntrack_tuple *orig, 221 const struct nf_conntrack_l3proto *l3proto, 222 const struct nf_conntrack_l4proto *l4proto) 223 { 224 memset(inverse, 0, sizeof(*inverse)); 225 226 inverse->src.l3num = orig->src.l3num; 227 if (l3proto->invert_tuple(inverse, orig) == 0) 228 return false; 229 230 inverse->dst.dir = !orig->dst.dir; 231 232 inverse->dst.protonum = orig->dst.protonum; 233 return l4proto->invert_tuple(inverse, orig); 234 } 235 EXPORT_SYMBOL_GPL(nf_ct_invert_tuple); 236 237 static void 238 clean_from_lists(struct nf_conn *ct) 239 { 240 pr_debug("clean_from_lists(%p)\n", ct); 241 hlist_nulls_del_rcu(&ct->tuplehash[IP_CT_DIR_ORIGINAL].hnnode); 242 hlist_nulls_del_rcu(&ct->tuplehash[IP_CT_DIR_REPLY].hnnode); 243 244 /* Destroy all pending expectations */ 245 nf_ct_remove_expectations(ct); 246 } 247 248 /* must be called with local_bh_disable */ 249 static void nf_ct_add_to_dying_list(struct nf_conn *ct) 250 { 251 struct ct_pcpu *pcpu; 252 253 /* add this conntrack to the (per cpu) dying list */ 254 ct->cpu = smp_processor_id(); 255 pcpu = per_cpu_ptr(nf_ct_net(ct)->ct.pcpu_lists, ct->cpu); 256 257 spin_lock(&pcpu->lock); 258 hlist_nulls_add_head(&ct->tuplehash[IP_CT_DIR_ORIGINAL].hnnode, 259 &pcpu->dying); 260 spin_unlock(&pcpu->lock); 261 } 262 263 /* must be called with local_bh_disable */ 264 static void nf_ct_add_to_unconfirmed_list(struct nf_conn *ct) 265 { 266 struct ct_pcpu *pcpu; 267 268 /* add this conntrack to the (per cpu) unconfirmed list */ 269 ct->cpu = smp_processor_id(); 270 pcpu = per_cpu_ptr(nf_ct_net(ct)->ct.pcpu_lists, ct->cpu); 271 272 spin_lock(&pcpu->lock); 273 hlist_nulls_add_head(&ct->tuplehash[IP_CT_DIR_ORIGINAL].hnnode, 274 &pcpu->unconfirmed); 275 spin_unlock(&pcpu->lock); 276 } 277 278 /* must be called with local_bh_disable */ 279 static void nf_ct_del_from_dying_or_unconfirmed_list(struct nf_conn *ct) 280 { 281 struct ct_pcpu *pcpu; 282 283 /* We overload first tuple to link into unconfirmed or dying list.*/ 284 pcpu = per_cpu_ptr(nf_ct_net(ct)->ct.pcpu_lists, ct->cpu); 285 286 spin_lock(&pcpu->lock); 287 BUG_ON(hlist_nulls_unhashed(&ct->tuplehash[IP_CT_DIR_ORIGINAL].hnnode)); 288 hlist_nulls_del_rcu(&ct->tuplehash[IP_CT_DIR_ORIGINAL].hnnode); 289 spin_unlock(&pcpu->lock); 290 } 291 292 /* Released via destroy_conntrack() */ 293 struct nf_conn *nf_ct_tmpl_alloc(struct net *net, 294 const struct nf_conntrack_zone *zone, 295 gfp_t flags) 296 { 297 struct nf_conn *tmpl; 298 299 tmpl = kzalloc(sizeof(*tmpl), flags); 300 if (tmpl == NULL) 301 return NULL; 302 303 tmpl->status = IPS_TEMPLATE; 304 write_pnet(&tmpl->ct_net, net); 305 306 if (nf_ct_zone_add(tmpl, flags, zone) < 0) 307 goto out_free; 308 309 atomic_set(&tmpl->ct_general.use, 0); 310 311 return tmpl; 312 out_free: 313 kfree(tmpl); 314 return NULL; 315 } 316 EXPORT_SYMBOL_GPL(nf_ct_tmpl_alloc); 317 318 void nf_ct_tmpl_free(struct nf_conn *tmpl) 319 { 320 nf_ct_ext_destroy(tmpl); 321 nf_ct_ext_free(tmpl); 322 kfree(tmpl); 323 } 324 EXPORT_SYMBOL_GPL(nf_ct_tmpl_free); 325 326 static void 327 destroy_conntrack(struct nf_conntrack *nfct) 328 { 329 struct nf_conn *ct = (struct nf_conn *)nfct; 330 struct net *net = nf_ct_net(ct); 331 struct nf_conntrack_l4proto *l4proto; 332 333 pr_debug("destroy_conntrack(%p)\n", ct); 334 NF_CT_ASSERT(atomic_read(&nfct->use) == 0); 335 NF_CT_ASSERT(!timer_pending(&ct->timeout)); 336 337 if (unlikely(nf_ct_is_template(ct))) { 338 nf_ct_tmpl_free(ct); 339 return; 340 } 341 rcu_read_lock(); 342 l4proto = __nf_ct_l4proto_find(nf_ct_l3num(ct), nf_ct_protonum(ct)); 343 if (l4proto && l4proto->destroy) 344 l4proto->destroy(ct); 345 346 rcu_read_unlock(); 347 348 local_bh_disable(); 349 /* Expectations will have been removed in clean_from_lists, 350 * except TFTP can create an expectation on the first packet, 351 * before connection is in the list, so we need to clean here, 352 * too. 353 */ 354 nf_ct_remove_expectations(ct); 355 356 nf_ct_del_from_dying_or_unconfirmed_list(ct); 357 358 NF_CT_STAT_INC(net, delete); 359 local_bh_enable(); 360 361 if (ct->master) 362 nf_ct_put(ct->master); 363 364 pr_debug("destroy_conntrack: returning ct=%p to slab\n", ct); 365 nf_conntrack_free(ct); 366 } 367 368 static void nf_ct_delete_from_lists(struct nf_conn *ct) 369 { 370 struct net *net = nf_ct_net(ct); 371 unsigned int hash, reply_hash; 372 unsigned int sequence; 373 374 nf_ct_helper_destroy(ct); 375 376 local_bh_disable(); 377 do { 378 sequence = read_seqcount_begin(&net->ct.generation); 379 hash = hash_conntrack(net, 380 &ct->tuplehash[IP_CT_DIR_ORIGINAL].tuple); 381 reply_hash = hash_conntrack(net, 382 &ct->tuplehash[IP_CT_DIR_REPLY].tuple); 383 } while (nf_conntrack_double_lock(net, hash, reply_hash, sequence)); 384 385 clean_from_lists(ct); 386 nf_conntrack_double_unlock(hash, reply_hash); 387 388 nf_ct_add_to_dying_list(ct); 389 390 NF_CT_STAT_INC(net, delete_list); 391 local_bh_enable(); 392 } 393 394 bool nf_ct_delete(struct nf_conn *ct, u32 portid, int report) 395 { 396 struct nf_conn_tstamp *tstamp; 397 398 tstamp = nf_conn_tstamp_find(ct); 399 if (tstamp && tstamp->stop == 0) 400 tstamp->stop = ktime_get_real_ns(); 401 402 if (nf_ct_is_dying(ct)) 403 goto delete; 404 405 if (nf_conntrack_event_report(IPCT_DESTROY, ct, 406 portid, report) < 0) { 407 /* destroy event was not delivered */ 408 nf_ct_delete_from_lists(ct); 409 nf_conntrack_ecache_delayed_work(nf_ct_net(ct)); 410 return false; 411 } 412 413 nf_conntrack_ecache_work(nf_ct_net(ct)); 414 set_bit(IPS_DYING_BIT, &ct->status); 415 delete: 416 nf_ct_delete_from_lists(ct); 417 nf_ct_put(ct); 418 return true; 419 } 420 EXPORT_SYMBOL_GPL(nf_ct_delete); 421 422 static void death_by_timeout(unsigned long ul_conntrack) 423 { 424 nf_ct_delete((struct nf_conn *)ul_conntrack, 0, 0); 425 } 426 427 static inline bool 428 nf_ct_key_equal(struct nf_conntrack_tuple_hash *h, 429 const struct nf_conntrack_tuple *tuple, 430 const struct nf_conntrack_zone *zone) 431 { 432 struct nf_conn *ct = nf_ct_tuplehash_to_ctrack(h); 433 434 /* A conntrack can be recreated with the equal tuple, 435 * so we need to check that the conntrack is confirmed 436 */ 437 return nf_ct_tuple_equal(tuple, &h->tuple) && 438 nf_ct_zone_equal(ct, zone, NF_CT_DIRECTION(h)) && 439 nf_ct_is_confirmed(ct); 440 } 441 442 /* 443 * Warning : 444 * - Caller must take a reference on returned object 445 * and recheck nf_ct_tuple_equal(tuple, &h->tuple) 446 */ 447 static struct nf_conntrack_tuple_hash * 448 ____nf_conntrack_find(struct net *net, const struct nf_conntrack_zone *zone, 449 const struct nf_conntrack_tuple *tuple, u32 hash) 450 { 451 struct nf_conntrack_tuple_hash *h; 452 struct hlist_nulls_node *n; 453 unsigned int bucket = hash_bucket(hash, net); 454 455 /* Disable BHs the entire time since we normally need to disable them 456 * at least once for the stats anyway. 457 */ 458 local_bh_disable(); 459 begin: 460 hlist_nulls_for_each_entry_rcu(h, n, &net->ct.hash[bucket], hnnode) { 461 if (nf_ct_key_equal(h, tuple, zone)) { 462 NF_CT_STAT_INC(net, found); 463 local_bh_enable(); 464 return h; 465 } 466 NF_CT_STAT_INC(net, searched); 467 } 468 /* 469 * if the nulls value we got at the end of this lookup is 470 * not the expected one, we must restart lookup. 471 * We probably met an item that was moved to another chain. 472 */ 473 if (get_nulls_value(n) != bucket) { 474 NF_CT_STAT_INC(net, search_restart); 475 goto begin; 476 } 477 local_bh_enable(); 478 479 return NULL; 480 } 481 482 /* Find a connection corresponding to a tuple. */ 483 static struct nf_conntrack_tuple_hash * 484 __nf_conntrack_find_get(struct net *net, const struct nf_conntrack_zone *zone, 485 const struct nf_conntrack_tuple *tuple, u32 hash) 486 { 487 struct nf_conntrack_tuple_hash *h; 488 struct nf_conn *ct; 489 490 rcu_read_lock(); 491 begin: 492 h = ____nf_conntrack_find(net, zone, tuple, hash); 493 if (h) { 494 ct = nf_ct_tuplehash_to_ctrack(h); 495 if (unlikely(nf_ct_is_dying(ct) || 496 !atomic_inc_not_zero(&ct->ct_general.use))) 497 h = NULL; 498 else { 499 if (unlikely(!nf_ct_key_equal(h, tuple, zone))) { 500 nf_ct_put(ct); 501 goto begin; 502 } 503 } 504 } 505 rcu_read_unlock(); 506 507 return h; 508 } 509 510 struct nf_conntrack_tuple_hash * 511 nf_conntrack_find_get(struct net *net, const struct nf_conntrack_zone *zone, 512 const struct nf_conntrack_tuple *tuple) 513 { 514 return __nf_conntrack_find_get(net, zone, tuple, 515 hash_conntrack_raw(tuple)); 516 } 517 EXPORT_SYMBOL_GPL(nf_conntrack_find_get); 518 519 static void __nf_conntrack_hash_insert(struct nf_conn *ct, 520 unsigned int hash, 521 unsigned int reply_hash) 522 { 523 struct net *net = nf_ct_net(ct); 524 525 hlist_nulls_add_head_rcu(&ct->tuplehash[IP_CT_DIR_ORIGINAL].hnnode, 526 &net->ct.hash[hash]); 527 hlist_nulls_add_head_rcu(&ct->tuplehash[IP_CT_DIR_REPLY].hnnode, 528 &net->ct.hash[reply_hash]); 529 } 530 531 int 532 nf_conntrack_hash_check_insert(struct nf_conn *ct) 533 { 534 const struct nf_conntrack_zone *zone; 535 struct net *net = nf_ct_net(ct); 536 unsigned int hash, reply_hash; 537 struct nf_conntrack_tuple_hash *h; 538 struct hlist_nulls_node *n; 539 unsigned int sequence; 540 541 zone = nf_ct_zone(ct); 542 543 local_bh_disable(); 544 do { 545 sequence = read_seqcount_begin(&net->ct.generation); 546 hash = hash_conntrack(net, 547 &ct->tuplehash[IP_CT_DIR_ORIGINAL].tuple); 548 reply_hash = hash_conntrack(net, 549 &ct->tuplehash[IP_CT_DIR_REPLY].tuple); 550 } while (nf_conntrack_double_lock(net, hash, reply_hash, sequence)); 551 552 /* See if there's one in the list already, including reverse */ 553 hlist_nulls_for_each_entry(h, n, &net->ct.hash[hash], hnnode) 554 if (nf_ct_tuple_equal(&ct->tuplehash[IP_CT_DIR_ORIGINAL].tuple, 555 &h->tuple) && 556 nf_ct_zone_equal(nf_ct_tuplehash_to_ctrack(h), zone, 557 NF_CT_DIRECTION(h))) 558 goto out; 559 hlist_nulls_for_each_entry(h, n, &net->ct.hash[reply_hash], hnnode) 560 if (nf_ct_tuple_equal(&ct->tuplehash[IP_CT_DIR_REPLY].tuple, 561 &h->tuple) && 562 nf_ct_zone_equal(nf_ct_tuplehash_to_ctrack(h), zone, 563 NF_CT_DIRECTION(h))) 564 goto out; 565 566 add_timer(&ct->timeout); 567 smp_wmb(); 568 /* The caller holds a reference to this object */ 569 atomic_set(&ct->ct_general.use, 2); 570 __nf_conntrack_hash_insert(ct, hash, reply_hash); 571 nf_conntrack_double_unlock(hash, reply_hash); 572 NF_CT_STAT_INC(net, insert); 573 local_bh_enable(); 574 return 0; 575 576 out: 577 nf_conntrack_double_unlock(hash, reply_hash); 578 NF_CT_STAT_INC(net, insert_failed); 579 local_bh_enable(); 580 return -EEXIST; 581 } 582 EXPORT_SYMBOL_GPL(nf_conntrack_hash_check_insert); 583 584 /* Confirm a connection given skb; places it in hash table */ 585 int 586 __nf_conntrack_confirm(struct sk_buff *skb) 587 { 588 const struct nf_conntrack_zone *zone; 589 unsigned int hash, reply_hash; 590 struct nf_conntrack_tuple_hash *h; 591 struct nf_conn *ct; 592 struct nf_conn_help *help; 593 struct nf_conn_tstamp *tstamp; 594 struct hlist_nulls_node *n; 595 enum ip_conntrack_info ctinfo; 596 struct net *net; 597 unsigned int sequence; 598 599 ct = nf_ct_get(skb, &ctinfo); 600 net = nf_ct_net(ct); 601 602 /* ipt_REJECT uses nf_conntrack_attach to attach related 603 ICMP/TCP RST packets in other direction. Actual packet 604 which created connection will be IP_CT_NEW or for an 605 expected connection, IP_CT_RELATED. */ 606 if (CTINFO2DIR(ctinfo) != IP_CT_DIR_ORIGINAL) 607 return NF_ACCEPT; 608 609 zone = nf_ct_zone(ct); 610 local_bh_disable(); 611 612 do { 613 sequence = read_seqcount_begin(&net->ct.generation); 614 /* reuse the hash saved before */ 615 hash = *(unsigned long *)&ct->tuplehash[IP_CT_DIR_REPLY].hnnode.pprev; 616 hash = hash_bucket(hash, net); 617 reply_hash = hash_conntrack(net, 618 &ct->tuplehash[IP_CT_DIR_REPLY].tuple); 619 620 } while (nf_conntrack_double_lock(net, hash, reply_hash, sequence)); 621 622 /* We're not in hash table, and we refuse to set up related 623 * connections for unconfirmed conns. But packet copies and 624 * REJECT will give spurious warnings here. 625 */ 626 /* NF_CT_ASSERT(atomic_read(&ct->ct_general.use) == 1); */ 627 628 /* No external references means no one else could have 629 * confirmed us. 630 */ 631 NF_CT_ASSERT(!nf_ct_is_confirmed(ct)); 632 pr_debug("Confirming conntrack %p\n", ct); 633 /* We have to check the DYING flag after unlink to prevent 634 * a race against nf_ct_get_next_corpse() possibly called from 635 * user context, else we insert an already 'dead' hash, blocking 636 * further use of that particular connection -JM. 637 */ 638 nf_ct_del_from_dying_or_unconfirmed_list(ct); 639 640 if (unlikely(nf_ct_is_dying(ct))) 641 goto out; 642 643 /* See if there's one in the list already, including reverse: 644 NAT could have grabbed it without realizing, since we're 645 not in the hash. If there is, we lost race. */ 646 hlist_nulls_for_each_entry(h, n, &net->ct.hash[hash], hnnode) 647 if (nf_ct_tuple_equal(&ct->tuplehash[IP_CT_DIR_ORIGINAL].tuple, 648 &h->tuple) && 649 nf_ct_zone_equal(nf_ct_tuplehash_to_ctrack(h), zone, 650 NF_CT_DIRECTION(h))) 651 goto out; 652 hlist_nulls_for_each_entry(h, n, &net->ct.hash[reply_hash], hnnode) 653 if (nf_ct_tuple_equal(&ct->tuplehash[IP_CT_DIR_REPLY].tuple, 654 &h->tuple) && 655 nf_ct_zone_equal(nf_ct_tuplehash_to_ctrack(h), zone, 656 NF_CT_DIRECTION(h))) 657 goto out; 658 659 /* Timer relative to confirmation time, not original 660 setting time, otherwise we'd get timer wrap in 661 weird delay cases. */ 662 ct->timeout.expires += jiffies; 663 add_timer(&ct->timeout); 664 atomic_inc(&ct->ct_general.use); 665 ct->status |= IPS_CONFIRMED; 666 667 /* set conntrack timestamp, if enabled. */ 668 tstamp = nf_conn_tstamp_find(ct); 669 if (tstamp) { 670 if (skb->tstamp.tv64 == 0) 671 __net_timestamp(skb); 672 673 tstamp->start = ktime_to_ns(skb->tstamp); 674 } 675 /* Since the lookup is lockless, hash insertion must be done after 676 * starting the timer and setting the CONFIRMED bit. The RCU barriers 677 * guarantee that no other CPU can find the conntrack before the above 678 * stores are visible. 679 */ 680 __nf_conntrack_hash_insert(ct, hash, reply_hash); 681 nf_conntrack_double_unlock(hash, reply_hash); 682 NF_CT_STAT_INC(net, insert); 683 local_bh_enable(); 684 685 help = nfct_help(ct); 686 if (help && help->helper) 687 nf_conntrack_event_cache(IPCT_HELPER, ct); 688 689 nf_conntrack_event_cache(master_ct(ct) ? 690 IPCT_RELATED : IPCT_NEW, ct); 691 return NF_ACCEPT; 692 693 out: 694 nf_ct_add_to_dying_list(ct); 695 nf_conntrack_double_unlock(hash, reply_hash); 696 NF_CT_STAT_INC(net, insert_failed); 697 local_bh_enable(); 698 return NF_DROP; 699 } 700 EXPORT_SYMBOL_GPL(__nf_conntrack_confirm); 701 702 /* Returns true if a connection correspondings to the tuple (required 703 for NAT). */ 704 int 705 nf_conntrack_tuple_taken(const struct nf_conntrack_tuple *tuple, 706 const struct nf_conn *ignored_conntrack) 707 { 708 struct net *net = nf_ct_net(ignored_conntrack); 709 const struct nf_conntrack_zone *zone; 710 struct nf_conntrack_tuple_hash *h; 711 struct hlist_nulls_node *n; 712 struct nf_conn *ct; 713 unsigned int hash; 714 715 zone = nf_ct_zone(ignored_conntrack); 716 hash = hash_conntrack(net, tuple); 717 718 /* Disable BHs the entire time since we need to disable them at 719 * least once for the stats anyway. 720 */ 721 rcu_read_lock_bh(); 722 hlist_nulls_for_each_entry_rcu(h, n, &net->ct.hash[hash], hnnode) { 723 ct = nf_ct_tuplehash_to_ctrack(h); 724 if (ct != ignored_conntrack && 725 nf_ct_tuple_equal(tuple, &h->tuple) && 726 nf_ct_zone_equal(ct, zone, NF_CT_DIRECTION(h))) { 727 NF_CT_STAT_INC(net, found); 728 rcu_read_unlock_bh(); 729 return 1; 730 } 731 NF_CT_STAT_INC(net, searched); 732 } 733 rcu_read_unlock_bh(); 734 735 return 0; 736 } 737 EXPORT_SYMBOL_GPL(nf_conntrack_tuple_taken); 738 739 #define NF_CT_EVICTION_RANGE 8 740 741 /* There's a small race here where we may free a just-assured 742 connection. Too bad: we're in trouble anyway. */ 743 static noinline int early_drop(struct net *net, unsigned int _hash) 744 { 745 /* Use oldest entry, which is roughly LRU */ 746 struct nf_conntrack_tuple_hash *h; 747 struct nf_conn *ct = NULL, *tmp; 748 struct hlist_nulls_node *n; 749 unsigned int i = 0, cnt = 0; 750 int dropped = 0; 751 unsigned int hash, sequence; 752 spinlock_t *lockp; 753 754 local_bh_disable(); 755 restart: 756 sequence = read_seqcount_begin(&net->ct.generation); 757 hash = hash_bucket(_hash, net); 758 for (; i < net->ct.htable_size; i++) { 759 lockp = &nf_conntrack_locks[hash % CONNTRACK_LOCKS]; 760 spin_lock(lockp); 761 if (read_seqcount_retry(&net->ct.generation, sequence)) { 762 spin_unlock(lockp); 763 goto restart; 764 } 765 hlist_nulls_for_each_entry_rcu(h, n, &net->ct.hash[hash], 766 hnnode) { 767 tmp = nf_ct_tuplehash_to_ctrack(h); 768 if (!test_bit(IPS_ASSURED_BIT, &tmp->status) && 769 !nf_ct_is_dying(tmp) && 770 atomic_inc_not_zero(&tmp->ct_general.use)) { 771 ct = tmp; 772 break; 773 } 774 cnt++; 775 } 776 777 hash = (hash + 1) % net->ct.htable_size; 778 spin_unlock(lockp); 779 780 if (ct || cnt >= NF_CT_EVICTION_RANGE) 781 break; 782 783 } 784 local_bh_enable(); 785 786 if (!ct) 787 return dropped; 788 789 if (del_timer(&ct->timeout)) { 790 if (nf_ct_delete(ct, 0, 0)) { 791 dropped = 1; 792 NF_CT_STAT_INC_ATOMIC(net, early_drop); 793 } 794 } 795 nf_ct_put(ct); 796 return dropped; 797 } 798 799 void init_nf_conntrack_hash_rnd(void) 800 { 801 unsigned int rand; 802 803 /* 804 * Why not initialize nf_conntrack_rnd in a "init()" function ? 805 * Because there isn't enough entropy when system initializing, 806 * and we initialize it as late as possible. 807 */ 808 do { 809 get_random_bytes(&rand, sizeof(rand)); 810 } while (!rand); 811 cmpxchg(&nf_conntrack_hash_rnd, 0, rand); 812 } 813 814 static struct nf_conn * 815 __nf_conntrack_alloc(struct net *net, 816 const struct nf_conntrack_zone *zone, 817 const struct nf_conntrack_tuple *orig, 818 const struct nf_conntrack_tuple *repl, 819 gfp_t gfp, u32 hash) 820 { 821 struct nf_conn *ct; 822 823 if (unlikely(!nf_conntrack_hash_rnd)) { 824 init_nf_conntrack_hash_rnd(); 825 /* recompute the hash as nf_conntrack_hash_rnd is initialized */ 826 hash = hash_conntrack_raw(orig); 827 } 828 829 /* We don't want any race condition at early drop stage */ 830 atomic_inc(&net->ct.count); 831 832 if (nf_conntrack_max && 833 unlikely(atomic_read(&net->ct.count) > nf_conntrack_max)) { 834 if (!early_drop(net, hash)) { 835 atomic_dec(&net->ct.count); 836 net_warn_ratelimited("nf_conntrack: table full, dropping packet\n"); 837 return ERR_PTR(-ENOMEM); 838 } 839 } 840 841 /* 842 * Do not use kmem_cache_zalloc(), as this cache uses 843 * SLAB_DESTROY_BY_RCU. 844 */ 845 ct = kmem_cache_alloc(net->ct.nf_conntrack_cachep, gfp); 846 if (ct == NULL) 847 goto out; 848 849 spin_lock_init(&ct->lock); 850 ct->tuplehash[IP_CT_DIR_ORIGINAL].tuple = *orig; 851 ct->tuplehash[IP_CT_DIR_ORIGINAL].hnnode.pprev = NULL; 852 ct->tuplehash[IP_CT_DIR_REPLY].tuple = *repl; 853 /* save hash for reusing when confirming */ 854 *(unsigned long *)(&ct->tuplehash[IP_CT_DIR_REPLY].hnnode.pprev) = hash; 855 ct->status = 0; 856 /* Don't set timer yet: wait for confirmation */ 857 setup_timer(&ct->timeout, death_by_timeout, (unsigned long)ct); 858 write_pnet(&ct->ct_net, net); 859 memset(&ct->__nfct_init_offset[0], 0, 860 offsetof(struct nf_conn, proto) - 861 offsetof(struct nf_conn, __nfct_init_offset[0])); 862 863 if (zone && nf_ct_zone_add(ct, GFP_ATOMIC, zone) < 0) 864 goto out_free; 865 866 /* Because we use RCU lookups, we set ct_general.use to zero before 867 * this is inserted in any list. 868 */ 869 atomic_set(&ct->ct_general.use, 0); 870 return ct; 871 out_free: 872 kmem_cache_free(net->ct.nf_conntrack_cachep, ct); 873 out: 874 atomic_dec(&net->ct.count); 875 return ERR_PTR(-ENOMEM); 876 } 877 878 struct nf_conn *nf_conntrack_alloc(struct net *net, 879 const struct nf_conntrack_zone *zone, 880 const struct nf_conntrack_tuple *orig, 881 const struct nf_conntrack_tuple *repl, 882 gfp_t gfp) 883 { 884 return __nf_conntrack_alloc(net, zone, orig, repl, gfp, 0); 885 } 886 EXPORT_SYMBOL_GPL(nf_conntrack_alloc); 887 888 void nf_conntrack_free(struct nf_conn *ct) 889 { 890 struct net *net = nf_ct_net(ct); 891 892 /* A freed object has refcnt == 0, that's 893 * the golden rule for SLAB_DESTROY_BY_RCU 894 */ 895 NF_CT_ASSERT(atomic_read(&ct->ct_general.use) == 0); 896 897 nf_ct_ext_destroy(ct); 898 nf_ct_ext_free(ct); 899 kmem_cache_free(net->ct.nf_conntrack_cachep, ct); 900 smp_mb__before_atomic(); 901 atomic_dec(&net->ct.count); 902 } 903 EXPORT_SYMBOL_GPL(nf_conntrack_free); 904 905 906 /* Allocate a new conntrack: we return -ENOMEM if classification 907 failed due to stress. Otherwise it really is unclassifiable. */ 908 static struct nf_conntrack_tuple_hash * 909 init_conntrack(struct net *net, struct nf_conn *tmpl, 910 const struct nf_conntrack_tuple *tuple, 911 struct nf_conntrack_l3proto *l3proto, 912 struct nf_conntrack_l4proto *l4proto, 913 struct sk_buff *skb, 914 unsigned int dataoff, u32 hash) 915 { 916 struct nf_conn *ct; 917 struct nf_conn_help *help; 918 struct nf_conntrack_tuple repl_tuple; 919 struct nf_conntrack_ecache *ecache; 920 struct nf_conntrack_expect *exp = NULL; 921 const struct nf_conntrack_zone *zone; 922 struct nf_conn_timeout *timeout_ext; 923 struct nf_conntrack_zone tmp; 924 unsigned int *timeouts; 925 926 if (!nf_ct_invert_tuple(&repl_tuple, tuple, l3proto, l4proto)) { 927 pr_debug("Can't invert tuple.\n"); 928 return NULL; 929 } 930 931 zone = nf_ct_zone_tmpl(tmpl, skb, &tmp); 932 ct = __nf_conntrack_alloc(net, zone, tuple, &repl_tuple, GFP_ATOMIC, 933 hash); 934 if (IS_ERR(ct)) 935 return (struct nf_conntrack_tuple_hash *)ct; 936 937 if (tmpl && nfct_synproxy(tmpl)) { 938 nfct_seqadj_ext_add(ct); 939 nfct_synproxy_ext_add(ct); 940 } 941 942 timeout_ext = tmpl ? nf_ct_timeout_find(tmpl) : NULL; 943 if (timeout_ext) { 944 timeouts = nf_ct_timeout_data(timeout_ext); 945 if (unlikely(!timeouts)) 946 timeouts = l4proto->get_timeouts(net); 947 } else { 948 timeouts = l4proto->get_timeouts(net); 949 } 950 951 if (!l4proto->new(ct, skb, dataoff, timeouts)) { 952 nf_conntrack_free(ct); 953 pr_debug("init conntrack: can't track with proto module\n"); 954 return NULL; 955 } 956 957 if (timeout_ext) 958 nf_ct_timeout_ext_add(ct, rcu_dereference(timeout_ext->timeout), 959 GFP_ATOMIC); 960 961 nf_ct_acct_ext_add(ct, GFP_ATOMIC); 962 nf_ct_tstamp_ext_add(ct, GFP_ATOMIC); 963 nf_ct_labels_ext_add(ct); 964 965 ecache = tmpl ? nf_ct_ecache_find(tmpl) : NULL; 966 nf_ct_ecache_ext_add(ct, ecache ? ecache->ctmask : 0, 967 ecache ? ecache->expmask : 0, 968 GFP_ATOMIC); 969 970 local_bh_disable(); 971 if (net->ct.expect_count) { 972 spin_lock(&nf_conntrack_expect_lock); 973 exp = nf_ct_find_expectation(net, zone, tuple); 974 if (exp) { 975 pr_debug("conntrack: expectation arrives ct=%p exp=%p\n", 976 ct, exp); 977 /* Welcome, Mr. Bond. We've been expecting you... */ 978 __set_bit(IPS_EXPECTED_BIT, &ct->status); 979 /* exp->master safe, refcnt bumped in nf_ct_find_expectation */ 980 ct->master = exp->master; 981 if (exp->helper) { 982 help = nf_ct_helper_ext_add(ct, exp->helper, 983 GFP_ATOMIC); 984 if (help) 985 rcu_assign_pointer(help->helper, exp->helper); 986 } 987 988 #ifdef CONFIG_NF_CONNTRACK_MARK 989 ct->mark = exp->master->mark; 990 #endif 991 #ifdef CONFIG_NF_CONNTRACK_SECMARK 992 ct->secmark = exp->master->secmark; 993 #endif 994 NF_CT_STAT_INC(net, expect_new); 995 } 996 spin_unlock(&nf_conntrack_expect_lock); 997 } 998 if (!exp) { 999 __nf_ct_try_assign_helper(ct, tmpl, GFP_ATOMIC); 1000 NF_CT_STAT_INC(net, new); 1001 } 1002 1003 /* Now it is inserted into the unconfirmed list, bump refcount */ 1004 nf_conntrack_get(&ct->ct_general); 1005 nf_ct_add_to_unconfirmed_list(ct); 1006 1007 local_bh_enable(); 1008 1009 if (exp) { 1010 if (exp->expectfn) 1011 exp->expectfn(ct, exp); 1012 nf_ct_expect_put(exp); 1013 } 1014 1015 return &ct->tuplehash[IP_CT_DIR_ORIGINAL]; 1016 } 1017 1018 /* On success, returns conntrack ptr, sets skb->nfct and ctinfo */ 1019 static inline struct nf_conn * 1020 resolve_normal_ct(struct net *net, struct nf_conn *tmpl, 1021 struct sk_buff *skb, 1022 unsigned int dataoff, 1023 u_int16_t l3num, 1024 u_int8_t protonum, 1025 struct nf_conntrack_l3proto *l3proto, 1026 struct nf_conntrack_l4proto *l4proto, 1027 int *set_reply, 1028 enum ip_conntrack_info *ctinfo) 1029 { 1030 const struct nf_conntrack_zone *zone; 1031 struct nf_conntrack_tuple tuple; 1032 struct nf_conntrack_tuple_hash *h; 1033 struct nf_conntrack_zone tmp; 1034 struct nf_conn *ct; 1035 u32 hash; 1036 1037 if (!nf_ct_get_tuple(skb, skb_network_offset(skb), 1038 dataoff, l3num, protonum, net, &tuple, l3proto, 1039 l4proto)) { 1040 pr_debug("resolve_normal_ct: Can't get tuple\n"); 1041 return NULL; 1042 } 1043 1044 /* look for tuple match */ 1045 zone = nf_ct_zone_tmpl(tmpl, skb, &tmp); 1046 hash = hash_conntrack_raw(&tuple); 1047 h = __nf_conntrack_find_get(net, zone, &tuple, hash); 1048 if (!h) { 1049 h = init_conntrack(net, tmpl, &tuple, l3proto, l4proto, 1050 skb, dataoff, hash); 1051 if (!h) 1052 return NULL; 1053 if (IS_ERR(h)) 1054 return (void *)h; 1055 } 1056 ct = nf_ct_tuplehash_to_ctrack(h); 1057 1058 /* It exists; we have (non-exclusive) reference. */ 1059 if (NF_CT_DIRECTION(h) == IP_CT_DIR_REPLY) { 1060 *ctinfo = IP_CT_ESTABLISHED_REPLY; 1061 /* Please set reply bit if this packet OK */ 1062 *set_reply = 1; 1063 } else { 1064 /* Once we've had two way comms, always ESTABLISHED. */ 1065 if (test_bit(IPS_SEEN_REPLY_BIT, &ct->status)) { 1066 pr_debug("nf_conntrack_in: normal packet for %p\n", ct); 1067 *ctinfo = IP_CT_ESTABLISHED; 1068 } else if (test_bit(IPS_EXPECTED_BIT, &ct->status)) { 1069 pr_debug("nf_conntrack_in: related packet for %p\n", 1070 ct); 1071 *ctinfo = IP_CT_RELATED; 1072 } else { 1073 pr_debug("nf_conntrack_in: new packet for %p\n", ct); 1074 *ctinfo = IP_CT_NEW; 1075 } 1076 *set_reply = 0; 1077 } 1078 skb->nfct = &ct->ct_general; 1079 skb->nfctinfo = *ctinfo; 1080 return ct; 1081 } 1082 1083 unsigned int 1084 nf_conntrack_in(struct net *net, u_int8_t pf, unsigned int hooknum, 1085 struct sk_buff *skb) 1086 { 1087 struct nf_conn *ct, *tmpl = NULL; 1088 enum ip_conntrack_info ctinfo; 1089 struct nf_conntrack_l3proto *l3proto; 1090 struct nf_conntrack_l4proto *l4proto; 1091 unsigned int *timeouts; 1092 unsigned int dataoff; 1093 u_int8_t protonum; 1094 int set_reply = 0; 1095 int ret; 1096 1097 if (skb->nfct) { 1098 /* Previously seen (loopback or untracked)? Ignore. */ 1099 tmpl = (struct nf_conn *)skb->nfct; 1100 if (!nf_ct_is_template(tmpl)) { 1101 NF_CT_STAT_INC_ATOMIC(net, ignore); 1102 return NF_ACCEPT; 1103 } 1104 skb->nfct = NULL; 1105 } 1106 1107 /* rcu_read_lock()ed by nf_hook_slow */ 1108 l3proto = __nf_ct_l3proto_find(pf); 1109 ret = l3proto->get_l4proto(skb, skb_network_offset(skb), 1110 &dataoff, &protonum); 1111 if (ret <= 0) { 1112 pr_debug("not prepared to track yet or error occurred\n"); 1113 NF_CT_STAT_INC_ATOMIC(net, error); 1114 NF_CT_STAT_INC_ATOMIC(net, invalid); 1115 ret = -ret; 1116 goto out; 1117 } 1118 1119 l4proto = __nf_ct_l4proto_find(pf, protonum); 1120 1121 /* It may be an special packet, error, unclean... 1122 * inverse of the return code tells to the netfilter 1123 * core what to do with the packet. */ 1124 if (l4proto->error != NULL) { 1125 ret = l4proto->error(net, tmpl, skb, dataoff, &ctinfo, 1126 pf, hooknum); 1127 if (ret <= 0) { 1128 NF_CT_STAT_INC_ATOMIC(net, error); 1129 NF_CT_STAT_INC_ATOMIC(net, invalid); 1130 ret = -ret; 1131 goto out; 1132 } 1133 /* ICMP[v6] protocol trackers may assign one conntrack. */ 1134 if (skb->nfct) 1135 goto out; 1136 } 1137 1138 ct = resolve_normal_ct(net, tmpl, skb, dataoff, pf, protonum, 1139 l3proto, l4proto, &set_reply, &ctinfo); 1140 if (!ct) { 1141 /* Not valid part of a connection */ 1142 NF_CT_STAT_INC_ATOMIC(net, invalid); 1143 ret = NF_ACCEPT; 1144 goto out; 1145 } 1146 1147 if (IS_ERR(ct)) { 1148 /* Too stressed to deal. */ 1149 NF_CT_STAT_INC_ATOMIC(net, drop); 1150 ret = NF_DROP; 1151 goto out; 1152 } 1153 1154 NF_CT_ASSERT(skb->nfct); 1155 1156 /* Decide what timeout policy we want to apply to this flow. */ 1157 timeouts = nf_ct_timeout_lookup(net, ct, l4proto); 1158 1159 ret = l4proto->packet(ct, skb, dataoff, ctinfo, pf, hooknum, timeouts); 1160 if (ret <= 0) { 1161 /* Invalid: inverse of the return code tells 1162 * the netfilter core what to do */ 1163 pr_debug("nf_conntrack_in: Can't track with proto module\n"); 1164 nf_conntrack_put(skb->nfct); 1165 skb->nfct = NULL; 1166 NF_CT_STAT_INC_ATOMIC(net, invalid); 1167 if (ret == -NF_DROP) 1168 NF_CT_STAT_INC_ATOMIC(net, drop); 1169 ret = -ret; 1170 goto out; 1171 } 1172 1173 if (set_reply && !test_and_set_bit(IPS_SEEN_REPLY_BIT, &ct->status)) 1174 nf_conntrack_event_cache(IPCT_REPLY, ct); 1175 out: 1176 if (tmpl) { 1177 /* Special case: we have to repeat this hook, assign the 1178 * template again to this packet. We assume that this packet 1179 * has no conntrack assigned. This is used by nf_ct_tcp. */ 1180 if (ret == NF_REPEAT) 1181 skb->nfct = (struct nf_conntrack *)tmpl; 1182 else 1183 nf_ct_put(tmpl); 1184 } 1185 1186 return ret; 1187 } 1188 EXPORT_SYMBOL_GPL(nf_conntrack_in); 1189 1190 bool nf_ct_invert_tuplepr(struct nf_conntrack_tuple *inverse, 1191 const struct nf_conntrack_tuple *orig) 1192 { 1193 bool ret; 1194 1195 rcu_read_lock(); 1196 ret = nf_ct_invert_tuple(inverse, orig, 1197 __nf_ct_l3proto_find(orig->src.l3num), 1198 __nf_ct_l4proto_find(orig->src.l3num, 1199 orig->dst.protonum)); 1200 rcu_read_unlock(); 1201 return ret; 1202 } 1203 EXPORT_SYMBOL_GPL(nf_ct_invert_tuplepr); 1204 1205 /* Alter reply tuple (maybe alter helper). This is for NAT, and is 1206 implicitly racy: see __nf_conntrack_confirm */ 1207 void nf_conntrack_alter_reply(struct nf_conn *ct, 1208 const struct nf_conntrack_tuple *newreply) 1209 { 1210 struct nf_conn_help *help = nfct_help(ct); 1211 1212 /* Should be unconfirmed, so not in hash table yet */ 1213 NF_CT_ASSERT(!nf_ct_is_confirmed(ct)); 1214 1215 pr_debug("Altering reply tuple of %p to ", ct); 1216 nf_ct_dump_tuple(newreply); 1217 1218 ct->tuplehash[IP_CT_DIR_REPLY].tuple = *newreply; 1219 if (ct->master || (help && !hlist_empty(&help->expectations))) 1220 return; 1221 1222 rcu_read_lock(); 1223 __nf_ct_try_assign_helper(ct, NULL, GFP_ATOMIC); 1224 rcu_read_unlock(); 1225 } 1226 EXPORT_SYMBOL_GPL(nf_conntrack_alter_reply); 1227 1228 /* Refresh conntrack for this many jiffies and do accounting if do_acct is 1 */ 1229 void __nf_ct_refresh_acct(struct nf_conn *ct, 1230 enum ip_conntrack_info ctinfo, 1231 const struct sk_buff *skb, 1232 unsigned long extra_jiffies, 1233 int do_acct) 1234 { 1235 NF_CT_ASSERT(ct->timeout.data == (unsigned long)ct); 1236 NF_CT_ASSERT(skb); 1237 1238 /* Only update if this is not a fixed timeout */ 1239 if (test_bit(IPS_FIXED_TIMEOUT_BIT, &ct->status)) 1240 goto acct; 1241 1242 /* If not in hash table, timer will not be active yet */ 1243 if (!nf_ct_is_confirmed(ct)) { 1244 ct->timeout.expires = extra_jiffies; 1245 } else { 1246 unsigned long newtime = jiffies + extra_jiffies; 1247 1248 /* Only update the timeout if the new timeout is at least 1249 HZ jiffies from the old timeout. Need del_timer for race 1250 avoidance (may already be dying). */ 1251 if (newtime - ct->timeout.expires >= HZ) 1252 mod_timer_pending(&ct->timeout, newtime); 1253 } 1254 1255 acct: 1256 if (do_acct) { 1257 struct nf_conn_acct *acct; 1258 1259 acct = nf_conn_acct_find(ct); 1260 if (acct) { 1261 struct nf_conn_counter *counter = acct->counter; 1262 1263 atomic64_inc(&counter[CTINFO2DIR(ctinfo)].packets); 1264 atomic64_add(skb->len, &counter[CTINFO2DIR(ctinfo)].bytes); 1265 } 1266 } 1267 } 1268 EXPORT_SYMBOL_GPL(__nf_ct_refresh_acct); 1269 1270 bool __nf_ct_kill_acct(struct nf_conn *ct, 1271 enum ip_conntrack_info ctinfo, 1272 const struct sk_buff *skb, 1273 int do_acct) 1274 { 1275 if (do_acct) { 1276 struct nf_conn_acct *acct; 1277 1278 acct = nf_conn_acct_find(ct); 1279 if (acct) { 1280 struct nf_conn_counter *counter = acct->counter; 1281 1282 atomic64_inc(&counter[CTINFO2DIR(ctinfo)].packets); 1283 atomic64_add(skb->len - skb_network_offset(skb), 1284 &counter[CTINFO2DIR(ctinfo)].bytes); 1285 } 1286 } 1287 1288 if (del_timer(&ct->timeout)) { 1289 ct->timeout.function((unsigned long)ct); 1290 return true; 1291 } 1292 return false; 1293 } 1294 EXPORT_SYMBOL_GPL(__nf_ct_kill_acct); 1295 1296 #ifdef CONFIG_NF_CONNTRACK_ZONES 1297 static struct nf_ct_ext_type nf_ct_zone_extend __read_mostly = { 1298 .len = sizeof(struct nf_conntrack_zone), 1299 .align = __alignof__(struct nf_conntrack_zone), 1300 .id = NF_CT_EXT_ZONE, 1301 }; 1302 #endif 1303 1304 #if IS_ENABLED(CONFIG_NF_CT_NETLINK) 1305 1306 #include <linux/netfilter/nfnetlink.h> 1307 #include <linux/netfilter/nfnetlink_conntrack.h> 1308 #include <linux/mutex.h> 1309 1310 /* Generic function for tcp/udp/sctp/dccp and alike. This needs to be 1311 * in ip_conntrack_core, since we don't want the protocols to autoload 1312 * or depend on ctnetlink */ 1313 int nf_ct_port_tuple_to_nlattr(struct sk_buff *skb, 1314 const struct nf_conntrack_tuple *tuple) 1315 { 1316 if (nla_put_be16(skb, CTA_PROTO_SRC_PORT, tuple->src.u.tcp.port) || 1317 nla_put_be16(skb, CTA_PROTO_DST_PORT, tuple->dst.u.tcp.port)) 1318 goto nla_put_failure; 1319 return 0; 1320 1321 nla_put_failure: 1322 return -1; 1323 } 1324 EXPORT_SYMBOL_GPL(nf_ct_port_tuple_to_nlattr); 1325 1326 const struct nla_policy nf_ct_port_nla_policy[CTA_PROTO_MAX+1] = { 1327 [CTA_PROTO_SRC_PORT] = { .type = NLA_U16 }, 1328 [CTA_PROTO_DST_PORT] = { .type = NLA_U16 }, 1329 }; 1330 EXPORT_SYMBOL_GPL(nf_ct_port_nla_policy); 1331 1332 int nf_ct_port_nlattr_to_tuple(struct nlattr *tb[], 1333 struct nf_conntrack_tuple *t) 1334 { 1335 if (!tb[CTA_PROTO_SRC_PORT] || !tb[CTA_PROTO_DST_PORT]) 1336 return -EINVAL; 1337 1338 t->src.u.tcp.port = nla_get_be16(tb[CTA_PROTO_SRC_PORT]); 1339 t->dst.u.tcp.port = nla_get_be16(tb[CTA_PROTO_DST_PORT]); 1340 1341 return 0; 1342 } 1343 EXPORT_SYMBOL_GPL(nf_ct_port_nlattr_to_tuple); 1344 1345 int nf_ct_port_nlattr_tuple_size(void) 1346 { 1347 return nla_policy_len(nf_ct_port_nla_policy, CTA_PROTO_MAX + 1); 1348 } 1349 EXPORT_SYMBOL_GPL(nf_ct_port_nlattr_tuple_size); 1350 #endif 1351 1352 /* Used by ipt_REJECT and ip6t_REJECT. */ 1353 static void nf_conntrack_attach(struct sk_buff *nskb, const struct sk_buff *skb) 1354 { 1355 struct nf_conn *ct; 1356 enum ip_conntrack_info ctinfo; 1357 1358 /* This ICMP is in reverse direction to the packet which caused it */ 1359 ct = nf_ct_get(skb, &ctinfo); 1360 if (CTINFO2DIR(ctinfo) == IP_CT_DIR_ORIGINAL) 1361 ctinfo = IP_CT_RELATED_REPLY; 1362 else 1363 ctinfo = IP_CT_RELATED; 1364 1365 /* Attach to new skbuff, and increment count */ 1366 nskb->nfct = &ct->ct_general; 1367 nskb->nfctinfo = ctinfo; 1368 nf_conntrack_get(nskb->nfct); 1369 } 1370 1371 /* Bring out ya dead! */ 1372 static struct nf_conn * 1373 get_next_corpse(struct net *net, int (*iter)(struct nf_conn *i, void *data), 1374 void *data, unsigned int *bucket) 1375 { 1376 struct nf_conntrack_tuple_hash *h; 1377 struct nf_conn *ct; 1378 struct hlist_nulls_node *n; 1379 int cpu; 1380 spinlock_t *lockp; 1381 1382 for (; *bucket < net->ct.htable_size; (*bucket)++) { 1383 lockp = &nf_conntrack_locks[*bucket % CONNTRACK_LOCKS]; 1384 local_bh_disable(); 1385 spin_lock(lockp); 1386 if (*bucket < net->ct.htable_size) { 1387 hlist_nulls_for_each_entry(h, n, &net->ct.hash[*bucket], hnnode) { 1388 if (NF_CT_DIRECTION(h) != IP_CT_DIR_ORIGINAL) 1389 continue; 1390 ct = nf_ct_tuplehash_to_ctrack(h); 1391 if (iter(ct, data)) 1392 goto found; 1393 } 1394 } 1395 spin_unlock(lockp); 1396 local_bh_enable(); 1397 } 1398 1399 for_each_possible_cpu(cpu) { 1400 struct ct_pcpu *pcpu = per_cpu_ptr(net->ct.pcpu_lists, cpu); 1401 1402 spin_lock_bh(&pcpu->lock); 1403 hlist_nulls_for_each_entry(h, n, &pcpu->unconfirmed, hnnode) { 1404 ct = nf_ct_tuplehash_to_ctrack(h); 1405 if (iter(ct, data)) 1406 set_bit(IPS_DYING_BIT, &ct->status); 1407 } 1408 spin_unlock_bh(&pcpu->lock); 1409 } 1410 return NULL; 1411 found: 1412 atomic_inc(&ct->ct_general.use); 1413 spin_unlock(lockp); 1414 local_bh_enable(); 1415 return ct; 1416 } 1417 1418 void nf_ct_iterate_cleanup(struct net *net, 1419 int (*iter)(struct nf_conn *i, void *data), 1420 void *data, u32 portid, int report) 1421 { 1422 struct nf_conn *ct; 1423 unsigned int bucket = 0; 1424 1425 while ((ct = get_next_corpse(net, iter, data, &bucket)) != NULL) { 1426 /* Time to push up daises... */ 1427 if (del_timer(&ct->timeout)) 1428 nf_ct_delete(ct, portid, report); 1429 1430 /* ... else the timer will get him soon. */ 1431 1432 nf_ct_put(ct); 1433 } 1434 } 1435 EXPORT_SYMBOL_GPL(nf_ct_iterate_cleanup); 1436 1437 static int kill_all(struct nf_conn *i, void *data) 1438 { 1439 return 1; 1440 } 1441 1442 void nf_ct_free_hashtable(void *hash, unsigned int size) 1443 { 1444 if (is_vmalloc_addr(hash)) 1445 vfree(hash); 1446 else 1447 free_pages((unsigned long)hash, 1448 get_order(sizeof(struct hlist_head) * size)); 1449 } 1450 EXPORT_SYMBOL_GPL(nf_ct_free_hashtable); 1451 1452 static int untrack_refs(void) 1453 { 1454 int cnt = 0, cpu; 1455 1456 for_each_possible_cpu(cpu) { 1457 struct nf_conn *ct = &per_cpu(nf_conntrack_untracked, cpu); 1458 1459 cnt += atomic_read(&ct->ct_general.use) - 1; 1460 } 1461 return cnt; 1462 } 1463 1464 void nf_conntrack_cleanup_start(void) 1465 { 1466 RCU_INIT_POINTER(ip_ct_attach, NULL); 1467 } 1468 1469 void nf_conntrack_cleanup_end(void) 1470 { 1471 RCU_INIT_POINTER(nf_ct_destroy, NULL); 1472 while (untrack_refs() > 0) 1473 schedule(); 1474 1475 #ifdef CONFIG_NF_CONNTRACK_ZONES 1476 nf_ct_extend_unregister(&nf_ct_zone_extend); 1477 #endif 1478 nf_conntrack_proto_fini(); 1479 nf_conntrack_seqadj_fini(); 1480 nf_conntrack_labels_fini(); 1481 nf_conntrack_helper_fini(); 1482 nf_conntrack_timeout_fini(); 1483 nf_conntrack_ecache_fini(); 1484 nf_conntrack_tstamp_fini(); 1485 nf_conntrack_acct_fini(); 1486 nf_conntrack_expect_fini(); 1487 } 1488 1489 /* 1490 * Mishearing the voices in his head, our hero wonders how he's 1491 * supposed to kill the mall. 1492 */ 1493 void nf_conntrack_cleanup_net(struct net *net) 1494 { 1495 LIST_HEAD(single); 1496 1497 list_add(&net->exit_list, &single); 1498 nf_conntrack_cleanup_net_list(&single); 1499 } 1500 1501 void nf_conntrack_cleanup_net_list(struct list_head *net_exit_list) 1502 { 1503 int busy; 1504 struct net *net; 1505 1506 /* 1507 * This makes sure all current packets have passed through 1508 * netfilter framework. Roll on, two-stage module 1509 * delete... 1510 */ 1511 synchronize_net(); 1512 i_see_dead_people: 1513 busy = 0; 1514 list_for_each_entry(net, net_exit_list, exit_list) { 1515 nf_ct_iterate_cleanup(net, kill_all, NULL, 0, 0); 1516 if (atomic_read(&net->ct.count) != 0) 1517 busy = 1; 1518 } 1519 if (busy) { 1520 schedule(); 1521 goto i_see_dead_people; 1522 } 1523 1524 list_for_each_entry(net, net_exit_list, exit_list) { 1525 nf_ct_free_hashtable(net->ct.hash, net->ct.htable_size); 1526 nf_conntrack_proto_pernet_fini(net); 1527 nf_conntrack_helper_pernet_fini(net); 1528 nf_conntrack_ecache_pernet_fini(net); 1529 nf_conntrack_tstamp_pernet_fini(net); 1530 nf_conntrack_acct_pernet_fini(net); 1531 nf_conntrack_expect_pernet_fini(net); 1532 kmem_cache_destroy(net->ct.nf_conntrack_cachep); 1533 kfree(net->ct.slabname); 1534 free_percpu(net->ct.stat); 1535 free_percpu(net->ct.pcpu_lists); 1536 } 1537 } 1538 1539 void *nf_ct_alloc_hashtable(unsigned int *sizep, int nulls) 1540 { 1541 struct hlist_nulls_head *hash; 1542 unsigned int nr_slots, i; 1543 size_t sz; 1544 1545 BUILD_BUG_ON(sizeof(struct hlist_nulls_head) != sizeof(struct hlist_head)); 1546 nr_slots = *sizep = roundup(*sizep, PAGE_SIZE / sizeof(struct hlist_nulls_head)); 1547 sz = nr_slots * sizeof(struct hlist_nulls_head); 1548 hash = (void *)__get_free_pages(GFP_KERNEL | __GFP_NOWARN | __GFP_ZERO, 1549 get_order(sz)); 1550 if (!hash) 1551 hash = vzalloc(sz); 1552 1553 if (hash && nulls) 1554 for (i = 0; i < nr_slots; i++) 1555 INIT_HLIST_NULLS_HEAD(&hash[i], i); 1556 1557 return hash; 1558 } 1559 EXPORT_SYMBOL_GPL(nf_ct_alloc_hashtable); 1560 1561 int nf_conntrack_set_hashsize(const char *val, struct kernel_param *kp) 1562 { 1563 int i, bucket, rc; 1564 unsigned int hashsize, old_size; 1565 struct hlist_nulls_head *hash, *old_hash; 1566 struct nf_conntrack_tuple_hash *h; 1567 struct nf_conn *ct; 1568 1569 if (current->nsproxy->net_ns != &init_net) 1570 return -EOPNOTSUPP; 1571 1572 /* On boot, we can set this without any fancy locking. */ 1573 if (!nf_conntrack_htable_size) 1574 return param_set_uint(val, kp); 1575 1576 rc = kstrtouint(val, 0, &hashsize); 1577 if (rc) 1578 return rc; 1579 if (!hashsize) 1580 return -EINVAL; 1581 1582 hash = nf_ct_alloc_hashtable(&hashsize, 1); 1583 if (!hash) 1584 return -ENOMEM; 1585 1586 local_bh_disable(); 1587 nf_conntrack_all_lock(); 1588 write_seqcount_begin(&init_net.ct.generation); 1589 1590 /* Lookups in the old hash might happen in parallel, which means we 1591 * might get false negatives during connection lookup. New connections 1592 * created because of a false negative won't make it into the hash 1593 * though since that required taking the locks. 1594 */ 1595 1596 for (i = 0; i < init_net.ct.htable_size; i++) { 1597 while (!hlist_nulls_empty(&init_net.ct.hash[i])) { 1598 h = hlist_nulls_entry(init_net.ct.hash[i].first, 1599 struct nf_conntrack_tuple_hash, hnnode); 1600 ct = nf_ct_tuplehash_to_ctrack(h); 1601 hlist_nulls_del_rcu(&h->hnnode); 1602 bucket = __hash_conntrack(&h->tuple, hashsize); 1603 hlist_nulls_add_head_rcu(&h->hnnode, &hash[bucket]); 1604 } 1605 } 1606 old_size = init_net.ct.htable_size; 1607 old_hash = init_net.ct.hash; 1608 1609 init_net.ct.htable_size = nf_conntrack_htable_size = hashsize; 1610 init_net.ct.hash = hash; 1611 1612 write_seqcount_end(&init_net.ct.generation); 1613 nf_conntrack_all_unlock(); 1614 local_bh_enable(); 1615 1616 nf_ct_free_hashtable(old_hash, old_size); 1617 return 0; 1618 } 1619 EXPORT_SYMBOL_GPL(nf_conntrack_set_hashsize); 1620 1621 module_param_call(hashsize, nf_conntrack_set_hashsize, param_get_uint, 1622 &nf_conntrack_htable_size, 0600); 1623 1624 void nf_ct_untracked_status_or(unsigned long bits) 1625 { 1626 int cpu; 1627 1628 for_each_possible_cpu(cpu) 1629 per_cpu(nf_conntrack_untracked, cpu).status |= bits; 1630 } 1631 EXPORT_SYMBOL_GPL(nf_ct_untracked_status_or); 1632 1633 int nf_conntrack_init_start(void) 1634 { 1635 int max_factor = 8; 1636 int i, ret, cpu; 1637 1638 for (i = 0; i < CONNTRACK_LOCKS; i++) 1639 spin_lock_init(&nf_conntrack_locks[i]); 1640 1641 if (!nf_conntrack_htable_size) { 1642 /* Idea from tcp.c: use 1/16384 of memory. 1643 * On i386: 32MB machine has 512 buckets. 1644 * >= 1GB machines have 16384 buckets. 1645 * >= 4GB machines have 65536 buckets. 1646 */ 1647 nf_conntrack_htable_size 1648 = (((totalram_pages << PAGE_SHIFT) / 16384) 1649 / sizeof(struct hlist_head)); 1650 if (totalram_pages > (4 * (1024 * 1024 * 1024 / PAGE_SIZE))) 1651 nf_conntrack_htable_size = 65536; 1652 else if (totalram_pages > (1024 * 1024 * 1024 / PAGE_SIZE)) 1653 nf_conntrack_htable_size = 16384; 1654 if (nf_conntrack_htable_size < 32) 1655 nf_conntrack_htable_size = 32; 1656 1657 /* Use a max. factor of four by default to get the same max as 1658 * with the old struct list_heads. When a table size is given 1659 * we use the old value of 8 to avoid reducing the max. 1660 * entries. */ 1661 max_factor = 4; 1662 } 1663 nf_conntrack_max = max_factor * nf_conntrack_htable_size; 1664 1665 printk(KERN_INFO "nf_conntrack version %s (%u buckets, %d max)\n", 1666 NF_CONNTRACK_VERSION, nf_conntrack_htable_size, 1667 nf_conntrack_max); 1668 1669 ret = nf_conntrack_expect_init(); 1670 if (ret < 0) 1671 goto err_expect; 1672 1673 ret = nf_conntrack_acct_init(); 1674 if (ret < 0) 1675 goto err_acct; 1676 1677 ret = nf_conntrack_tstamp_init(); 1678 if (ret < 0) 1679 goto err_tstamp; 1680 1681 ret = nf_conntrack_ecache_init(); 1682 if (ret < 0) 1683 goto err_ecache; 1684 1685 ret = nf_conntrack_timeout_init(); 1686 if (ret < 0) 1687 goto err_timeout; 1688 1689 ret = nf_conntrack_helper_init(); 1690 if (ret < 0) 1691 goto err_helper; 1692 1693 ret = nf_conntrack_labels_init(); 1694 if (ret < 0) 1695 goto err_labels; 1696 1697 ret = nf_conntrack_seqadj_init(); 1698 if (ret < 0) 1699 goto err_seqadj; 1700 1701 #ifdef CONFIG_NF_CONNTRACK_ZONES 1702 ret = nf_ct_extend_register(&nf_ct_zone_extend); 1703 if (ret < 0) 1704 goto err_extend; 1705 #endif 1706 ret = nf_conntrack_proto_init(); 1707 if (ret < 0) 1708 goto err_proto; 1709 1710 /* Set up fake conntrack: to never be deleted, not in any hashes */ 1711 for_each_possible_cpu(cpu) { 1712 struct nf_conn *ct = &per_cpu(nf_conntrack_untracked, cpu); 1713 write_pnet(&ct->ct_net, &init_net); 1714 atomic_set(&ct->ct_general.use, 1); 1715 } 1716 /* - and look it like as a confirmed connection */ 1717 nf_ct_untracked_status_or(IPS_CONFIRMED | IPS_UNTRACKED); 1718 return 0; 1719 1720 err_proto: 1721 #ifdef CONFIG_NF_CONNTRACK_ZONES 1722 nf_ct_extend_unregister(&nf_ct_zone_extend); 1723 err_extend: 1724 #endif 1725 nf_conntrack_seqadj_fini(); 1726 err_seqadj: 1727 nf_conntrack_labels_fini(); 1728 err_labels: 1729 nf_conntrack_helper_fini(); 1730 err_helper: 1731 nf_conntrack_timeout_fini(); 1732 err_timeout: 1733 nf_conntrack_ecache_fini(); 1734 err_ecache: 1735 nf_conntrack_tstamp_fini(); 1736 err_tstamp: 1737 nf_conntrack_acct_fini(); 1738 err_acct: 1739 nf_conntrack_expect_fini(); 1740 err_expect: 1741 return ret; 1742 } 1743 1744 void nf_conntrack_init_end(void) 1745 { 1746 /* For use by REJECT target */ 1747 RCU_INIT_POINTER(ip_ct_attach, nf_conntrack_attach); 1748 RCU_INIT_POINTER(nf_ct_destroy, destroy_conntrack); 1749 } 1750 1751 /* 1752 * We need to use special "null" values, not used in hash table 1753 */ 1754 #define UNCONFIRMED_NULLS_VAL ((1<<30)+0) 1755 #define DYING_NULLS_VAL ((1<<30)+1) 1756 #define TEMPLATE_NULLS_VAL ((1<<30)+2) 1757 1758 int nf_conntrack_init_net(struct net *net) 1759 { 1760 int ret = -ENOMEM; 1761 int cpu; 1762 1763 atomic_set(&net->ct.count, 0); 1764 seqcount_init(&net->ct.generation); 1765 1766 net->ct.pcpu_lists = alloc_percpu(struct ct_pcpu); 1767 if (!net->ct.pcpu_lists) 1768 goto err_stat; 1769 1770 for_each_possible_cpu(cpu) { 1771 struct ct_pcpu *pcpu = per_cpu_ptr(net->ct.pcpu_lists, cpu); 1772 1773 spin_lock_init(&pcpu->lock); 1774 INIT_HLIST_NULLS_HEAD(&pcpu->unconfirmed, UNCONFIRMED_NULLS_VAL); 1775 INIT_HLIST_NULLS_HEAD(&pcpu->dying, DYING_NULLS_VAL); 1776 } 1777 1778 net->ct.stat = alloc_percpu(struct ip_conntrack_stat); 1779 if (!net->ct.stat) 1780 goto err_pcpu_lists; 1781 1782 net->ct.slabname = kasprintf(GFP_KERNEL, "nf_conntrack_%p", net); 1783 if (!net->ct.slabname) 1784 goto err_slabname; 1785 1786 net->ct.nf_conntrack_cachep = kmem_cache_create(net->ct.slabname, 1787 sizeof(struct nf_conn), 0, 1788 SLAB_DESTROY_BY_RCU, NULL); 1789 if (!net->ct.nf_conntrack_cachep) { 1790 printk(KERN_ERR "Unable to create nf_conn slab cache\n"); 1791 goto err_cache; 1792 } 1793 1794 net->ct.htable_size = nf_conntrack_htable_size; 1795 net->ct.hash = nf_ct_alloc_hashtable(&net->ct.htable_size, 1); 1796 if (!net->ct.hash) { 1797 printk(KERN_ERR "Unable to create nf_conntrack_hash\n"); 1798 goto err_hash; 1799 } 1800 ret = nf_conntrack_expect_pernet_init(net); 1801 if (ret < 0) 1802 goto err_expect; 1803 ret = nf_conntrack_acct_pernet_init(net); 1804 if (ret < 0) 1805 goto err_acct; 1806 ret = nf_conntrack_tstamp_pernet_init(net); 1807 if (ret < 0) 1808 goto err_tstamp; 1809 ret = nf_conntrack_ecache_pernet_init(net); 1810 if (ret < 0) 1811 goto err_ecache; 1812 ret = nf_conntrack_helper_pernet_init(net); 1813 if (ret < 0) 1814 goto err_helper; 1815 ret = nf_conntrack_proto_pernet_init(net); 1816 if (ret < 0) 1817 goto err_proto; 1818 return 0; 1819 1820 err_proto: 1821 nf_conntrack_helper_pernet_fini(net); 1822 err_helper: 1823 nf_conntrack_ecache_pernet_fini(net); 1824 err_ecache: 1825 nf_conntrack_tstamp_pernet_fini(net); 1826 err_tstamp: 1827 nf_conntrack_acct_pernet_fini(net); 1828 err_acct: 1829 nf_conntrack_expect_pernet_fini(net); 1830 err_expect: 1831 nf_ct_free_hashtable(net->ct.hash, net->ct.htable_size); 1832 err_hash: 1833 kmem_cache_destroy(net->ct.nf_conntrack_cachep); 1834 err_cache: 1835 kfree(net->ct.slabname); 1836 err_slabname: 1837 free_percpu(net->ct.stat); 1838 err_pcpu_lists: 1839 free_percpu(net->ct.pcpu_lists); 1840 err_stat: 1841 return ret; 1842 } 1843