1 #include <linux/kernel.h> 2 #include <linux/init.h> 3 #include <linux/module.h> 4 #include <linux/netfilter.h> 5 #include <linux/rhashtable.h> 6 #include <linux/netdevice.h> 7 #include <net/ip.h> 8 #include <net/ip6_route.h> 9 #include <net/netfilter/nf_tables.h> 10 #include <net/netfilter/nf_flow_table.h> 11 #include <net/netfilter/nf_conntrack.h> 12 #include <net/netfilter/nf_conntrack_core.h> 13 #include <net/netfilter/nf_conntrack_tuple.h> 14 15 struct flow_offload_entry { 16 struct flow_offload flow; 17 struct nf_conn *ct; 18 struct rcu_head rcu_head; 19 }; 20 21 static DEFINE_MUTEX(flowtable_lock); 22 static LIST_HEAD(flowtables); 23 24 static void 25 flow_offload_fill_dir(struct flow_offload *flow, struct nf_conn *ct, 26 struct nf_flow_route *route, 27 enum flow_offload_tuple_dir dir) 28 { 29 struct flow_offload_tuple *ft = &flow->tuplehash[dir].tuple; 30 struct nf_conntrack_tuple *ctt = &ct->tuplehash[dir].tuple; 31 struct dst_entry *other_dst = route->tuple[!dir].dst; 32 struct dst_entry *dst = route->tuple[dir].dst; 33 34 ft->dir = dir; 35 36 switch (ctt->src.l3num) { 37 case NFPROTO_IPV4: 38 ft->src_v4 = ctt->src.u3.in; 39 ft->dst_v4 = ctt->dst.u3.in; 40 ft->mtu = ip_dst_mtu_maybe_forward(dst, true); 41 break; 42 case NFPROTO_IPV6: 43 ft->src_v6 = ctt->src.u3.in6; 44 ft->dst_v6 = ctt->dst.u3.in6; 45 ft->mtu = ip6_dst_mtu_forward(dst); 46 break; 47 } 48 49 ft->l3proto = ctt->src.l3num; 50 ft->l4proto = ctt->dst.protonum; 51 ft->src_port = ctt->src.u.tcp.port; 52 ft->dst_port = ctt->dst.u.tcp.port; 53 54 ft->iifidx = other_dst->dev->ifindex; 55 ft->oifidx = dst->dev->ifindex; 56 ft->dst_cache = dst; 57 } 58 59 struct flow_offload * 60 flow_offload_alloc(struct nf_conn *ct, struct nf_flow_route *route) 61 { 62 struct flow_offload_entry *entry; 63 struct flow_offload *flow; 64 65 if (unlikely(nf_ct_is_dying(ct) || 66 !atomic_inc_not_zero(&ct->ct_general.use))) 67 return NULL; 68 69 entry = kzalloc(sizeof(*entry), GFP_ATOMIC); 70 if (!entry) 71 goto err_ct_refcnt; 72 73 flow = &entry->flow; 74 75 if (!dst_hold_safe(route->tuple[FLOW_OFFLOAD_DIR_ORIGINAL].dst)) 76 goto err_dst_cache_original; 77 78 if (!dst_hold_safe(route->tuple[FLOW_OFFLOAD_DIR_REPLY].dst)) 79 goto err_dst_cache_reply; 80 81 entry->ct = ct; 82 83 flow_offload_fill_dir(flow, ct, route, FLOW_OFFLOAD_DIR_ORIGINAL); 84 flow_offload_fill_dir(flow, ct, route, FLOW_OFFLOAD_DIR_REPLY); 85 86 if (ct->status & IPS_SRC_NAT) 87 flow->flags |= FLOW_OFFLOAD_SNAT; 88 if (ct->status & IPS_DST_NAT) 89 flow->flags |= FLOW_OFFLOAD_DNAT; 90 91 return flow; 92 93 err_dst_cache_reply: 94 dst_release(route->tuple[FLOW_OFFLOAD_DIR_ORIGINAL].dst); 95 err_dst_cache_original: 96 kfree(entry); 97 err_ct_refcnt: 98 nf_ct_put(ct); 99 100 return NULL; 101 } 102 EXPORT_SYMBOL_GPL(flow_offload_alloc); 103 104 static void flow_offload_fixup_tcp(struct ip_ct_tcp *tcp) 105 { 106 tcp->state = TCP_CONNTRACK_ESTABLISHED; 107 tcp->seen[0].td_maxwin = 0; 108 tcp->seen[1].td_maxwin = 0; 109 } 110 111 #define NF_FLOWTABLE_TCP_PICKUP_TIMEOUT (120 * HZ) 112 #define NF_FLOWTABLE_UDP_PICKUP_TIMEOUT (30 * HZ) 113 114 static void flow_offload_fixup_ct_state(struct nf_conn *ct) 115 { 116 const struct nf_conntrack_l4proto *l4proto; 117 unsigned int timeout; 118 int l4num; 119 120 l4num = nf_ct_protonum(ct); 121 if (l4num == IPPROTO_TCP) 122 flow_offload_fixup_tcp(&ct->proto.tcp); 123 124 l4proto = nf_ct_l4proto_find(l4num); 125 if (!l4proto) 126 return; 127 128 if (l4num == IPPROTO_TCP) 129 timeout = NF_FLOWTABLE_TCP_PICKUP_TIMEOUT; 130 else if (l4num == IPPROTO_UDP) 131 timeout = NF_FLOWTABLE_UDP_PICKUP_TIMEOUT; 132 else 133 return; 134 135 ct->timeout = nfct_time_stamp + timeout; 136 } 137 138 void flow_offload_free(struct flow_offload *flow) 139 { 140 struct flow_offload_entry *e; 141 142 dst_release(flow->tuplehash[FLOW_OFFLOAD_DIR_ORIGINAL].tuple.dst_cache); 143 dst_release(flow->tuplehash[FLOW_OFFLOAD_DIR_REPLY].tuple.dst_cache); 144 e = container_of(flow, struct flow_offload_entry, flow); 145 if (flow->flags & FLOW_OFFLOAD_DYING) 146 nf_ct_delete(e->ct, 0, 0); 147 nf_ct_put(e->ct); 148 kfree_rcu(e, rcu_head); 149 } 150 EXPORT_SYMBOL_GPL(flow_offload_free); 151 152 static u32 flow_offload_hash(const void *data, u32 len, u32 seed) 153 { 154 const struct flow_offload_tuple *tuple = data; 155 156 return jhash(tuple, offsetof(struct flow_offload_tuple, dir), seed); 157 } 158 159 static u32 flow_offload_hash_obj(const void *data, u32 len, u32 seed) 160 { 161 const struct flow_offload_tuple_rhash *tuplehash = data; 162 163 return jhash(&tuplehash->tuple, offsetof(struct flow_offload_tuple, dir), seed); 164 } 165 166 static int flow_offload_hash_cmp(struct rhashtable_compare_arg *arg, 167 const void *ptr) 168 { 169 const struct flow_offload_tuple *tuple = arg->key; 170 const struct flow_offload_tuple_rhash *x = ptr; 171 172 if (memcmp(&x->tuple, tuple, offsetof(struct flow_offload_tuple, dir))) 173 return 1; 174 175 return 0; 176 } 177 178 static const struct rhashtable_params nf_flow_offload_rhash_params = { 179 .head_offset = offsetof(struct flow_offload_tuple_rhash, node), 180 .hashfn = flow_offload_hash, 181 .obj_hashfn = flow_offload_hash_obj, 182 .obj_cmpfn = flow_offload_hash_cmp, 183 .automatic_shrinking = true, 184 }; 185 186 int flow_offload_add(struct nf_flowtable *flow_table, struct flow_offload *flow) 187 { 188 flow->timeout = (u32)jiffies; 189 190 rhashtable_insert_fast(&flow_table->rhashtable, 191 &flow->tuplehash[FLOW_OFFLOAD_DIR_ORIGINAL].node, 192 nf_flow_offload_rhash_params); 193 rhashtable_insert_fast(&flow_table->rhashtable, 194 &flow->tuplehash[FLOW_OFFLOAD_DIR_REPLY].node, 195 nf_flow_offload_rhash_params); 196 return 0; 197 } 198 EXPORT_SYMBOL_GPL(flow_offload_add); 199 200 static void flow_offload_del(struct nf_flowtable *flow_table, 201 struct flow_offload *flow) 202 { 203 struct flow_offload_entry *e; 204 205 rhashtable_remove_fast(&flow_table->rhashtable, 206 &flow->tuplehash[FLOW_OFFLOAD_DIR_ORIGINAL].node, 207 nf_flow_offload_rhash_params); 208 rhashtable_remove_fast(&flow_table->rhashtable, 209 &flow->tuplehash[FLOW_OFFLOAD_DIR_REPLY].node, 210 nf_flow_offload_rhash_params); 211 212 e = container_of(flow, struct flow_offload_entry, flow); 213 clear_bit(IPS_OFFLOAD_BIT, &e->ct->status); 214 215 flow_offload_free(flow); 216 } 217 218 void flow_offload_teardown(struct flow_offload *flow) 219 { 220 struct flow_offload_entry *e; 221 222 flow->flags |= FLOW_OFFLOAD_TEARDOWN; 223 224 e = container_of(flow, struct flow_offload_entry, flow); 225 flow_offload_fixup_ct_state(e->ct); 226 } 227 EXPORT_SYMBOL_GPL(flow_offload_teardown); 228 229 struct flow_offload_tuple_rhash * 230 flow_offload_lookup(struct nf_flowtable *flow_table, 231 struct flow_offload_tuple *tuple) 232 { 233 struct flow_offload_tuple_rhash *tuplehash; 234 struct flow_offload *flow; 235 int dir; 236 237 tuplehash = rhashtable_lookup(&flow_table->rhashtable, tuple, 238 nf_flow_offload_rhash_params); 239 if (!tuplehash) 240 return NULL; 241 242 dir = tuplehash->tuple.dir; 243 flow = container_of(tuplehash, struct flow_offload, tuplehash[dir]); 244 if (flow->flags & (FLOW_OFFLOAD_DYING | FLOW_OFFLOAD_TEARDOWN)) 245 return NULL; 246 247 return tuplehash; 248 } 249 EXPORT_SYMBOL_GPL(flow_offload_lookup); 250 251 static int 252 nf_flow_table_iterate(struct nf_flowtable *flow_table, 253 void (*iter)(struct flow_offload *flow, void *data), 254 void *data) 255 { 256 struct flow_offload_tuple_rhash *tuplehash; 257 struct rhashtable_iter hti; 258 struct flow_offload *flow; 259 int err = 0; 260 261 rhashtable_walk_enter(&flow_table->rhashtable, &hti); 262 rhashtable_walk_start(&hti); 263 264 while ((tuplehash = rhashtable_walk_next(&hti))) { 265 if (IS_ERR(tuplehash)) { 266 if (PTR_ERR(tuplehash) != -EAGAIN) { 267 err = PTR_ERR(tuplehash); 268 break; 269 } 270 continue; 271 } 272 if (tuplehash->tuple.dir) 273 continue; 274 275 flow = container_of(tuplehash, struct flow_offload, tuplehash[0]); 276 277 iter(flow, data); 278 } 279 rhashtable_walk_stop(&hti); 280 rhashtable_walk_exit(&hti); 281 282 return err; 283 } 284 285 static inline bool nf_flow_has_expired(const struct flow_offload *flow) 286 { 287 return (__s32)(flow->timeout - (u32)jiffies) <= 0; 288 } 289 290 static void nf_flow_offload_gc_step(struct flow_offload *flow, void *data) 291 { 292 struct nf_flowtable *flow_table = data; 293 294 if (nf_flow_has_expired(flow) || 295 (flow->flags & (FLOW_OFFLOAD_DYING | FLOW_OFFLOAD_TEARDOWN))) 296 flow_offload_del(flow_table, flow); 297 } 298 299 static void nf_flow_offload_work_gc(struct work_struct *work) 300 { 301 struct nf_flowtable *flow_table; 302 303 flow_table = container_of(work, struct nf_flowtable, gc_work.work); 304 nf_flow_table_iterate(flow_table, nf_flow_offload_gc_step, flow_table); 305 queue_delayed_work(system_power_efficient_wq, &flow_table->gc_work, HZ); 306 } 307 308 static int nf_flow_nat_port_tcp(struct sk_buff *skb, unsigned int thoff, 309 __be16 port, __be16 new_port) 310 { 311 struct tcphdr *tcph; 312 313 if (!pskb_may_pull(skb, thoff + sizeof(*tcph)) || 314 skb_try_make_writable(skb, thoff + sizeof(*tcph))) 315 return -1; 316 317 tcph = (void *)(skb_network_header(skb) + thoff); 318 inet_proto_csum_replace2(&tcph->check, skb, port, new_port, true); 319 320 return 0; 321 } 322 323 static int nf_flow_nat_port_udp(struct sk_buff *skb, unsigned int thoff, 324 __be16 port, __be16 new_port) 325 { 326 struct udphdr *udph; 327 328 if (!pskb_may_pull(skb, thoff + sizeof(*udph)) || 329 skb_try_make_writable(skb, thoff + sizeof(*udph))) 330 return -1; 331 332 udph = (void *)(skb_network_header(skb) + thoff); 333 if (udph->check || skb->ip_summed == CHECKSUM_PARTIAL) { 334 inet_proto_csum_replace2(&udph->check, skb, port, 335 new_port, true); 336 if (!udph->check) 337 udph->check = CSUM_MANGLED_0; 338 } 339 340 return 0; 341 } 342 343 static int nf_flow_nat_port(struct sk_buff *skb, unsigned int thoff, 344 u8 protocol, __be16 port, __be16 new_port) 345 { 346 switch (protocol) { 347 case IPPROTO_TCP: 348 if (nf_flow_nat_port_tcp(skb, thoff, port, new_port) < 0) 349 return NF_DROP; 350 break; 351 case IPPROTO_UDP: 352 if (nf_flow_nat_port_udp(skb, thoff, port, new_port) < 0) 353 return NF_DROP; 354 break; 355 } 356 357 return 0; 358 } 359 360 int nf_flow_snat_port(const struct flow_offload *flow, 361 struct sk_buff *skb, unsigned int thoff, 362 u8 protocol, enum flow_offload_tuple_dir dir) 363 { 364 struct flow_ports *hdr; 365 __be16 port, new_port; 366 367 if (!pskb_may_pull(skb, thoff + sizeof(*hdr)) || 368 skb_try_make_writable(skb, thoff + sizeof(*hdr))) 369 return -1; 370 371 hdr = (void *)(skb_network_header(skb) + thoff); 372 373 switch (dir) { 374 case FLOW_OFFLOAD_DIR_ORIGINAL: 375 port = hdr->source; 376 new_port = flow->tuplehash[FLOW_OFFLOAD_DIR_REPLY].tuple.dst_port; 377 hdr->source = new_port; 378 break; 379 case FLOW_OFFLOAD_DIR_REPLY: 380 port = hdr->dest; 381 new_port = flow->tuplehash[FLOW_OFFLOAD_DIR_ORIGINAL].tuple.src_port; 382 hdr->dest = new_port; 383 break; 384 default: 385 return -1; 386 } 387 388 return nf_flow_nat_port(skb, thoff, protocol, port, new_port); 389 } 390 EXPORT_SYMBOL_GPL(nf_flow_snat_port); 391 392 int nf_flow_dnat_port(const struct flow_offload *flow, 393 struct sk_buff *skb, unsigned int thoff, 394 u8 protocol, enum flow_offload_tuple_dir dir) 395 { 396 struct flow_ports *hdr; 397 __be16 port, new_port; 398 399 if (!pskb_may_pull(skb, thoff + sizeof(*hdr)) || 400 skb_try_make_writable(skb, thoff + sizeof(*hdr))) 401 return -1; 402 403 hdr = (void *)(skb_network_header(skb) + thoff); 404 405 switch (dir) { 406 case FLOW_OFFLOAD_DIR_ORIGINAL: 407 port = hdr->dest; 408 new_port = flow->tuplehash[FLOW_OFFLOAD_DIR_REPLY].tuple.src_port; 409 hdr->dest = new_port; 410 break; 411 case FLOW_OFFLOAD_DIR_REPLY: 412 port = hdr->source; 413 new_port = flow->tuplehash[FLOW_OFFLOAD_DIR_ORIGINAL].tuple.dst_port; 414 hdr->source = new_port; 415 break; 416 default: 417 return -1; 418 } 419 420 return nf_flow_nat_port(skb, thoff, protocol, port, new_port); 421 } 422 EXPORT_SYMBOL_GPL(nf_flow_dnat_port); 423 424 int nf_flow_table_init(struct nf_flowtable *flowtable) 425 { 426 int err; 427 428 INIT_DEFERRABLE_WORK(&flowtable->gc_work, nf_flow_offload_work_gc); 429 430 err = rhashtable_init(&flowtable->rhashtable, 431 &nf_flow_offload_rhash_params); 432 if (err < 0) 433 return err; 434 435 queue_delayed_work(system_power_efficient_wq, 436 &flowtable->gc_work, HZ); 437 438 mutex_lock(&flowtable_lock); 439 list_add(&flowtable->list, &flowtables); 440 mutex_unlock(&flowtable_lock); 441 442 return 0; 443 } 444 EXPORT_SYMBOL_GPL(nf_flow_table_init); 445 446 static void nf_flow_table_do_cleanup(struct flow_offload *flow, void *data) 447 { 448 struct net_device *dev = data; 449 struct flow_offload_entry *e; 450 451 e = container_of(flow, struct flow_offload_entry, flow); 452 453 if (!dev) { 454 flow_offload_teardown(flow); 455 return; 456 } 457 if (net_eq(nf_ct_net(e->ct), dev_net(dev)) && 458 (flow->tuplehash[0].tuple.iifidx == dev->ifindex || 459 flow->tuplehash[1].tuple.iifidx == dev->ifindex)) 460 flow_offload_dead(flow); 461 } 462 463 static void nf_flow_table_iterate_cleanup(struct nf_flowtable *flowtable, 464 struct net_device *dev) 465 { 466 nf_flow_table_iterate(flowtable, nf_flow_table_do_cleanup, dev); 467 flush_delayed_work(&flowtable->gc_work); 468 } 469 470 void nf_flow_table_cleanup(struct net_device *dev) 471 { 472 struct nf_flowtable *flowtable; 473 474 mutex_lock(&flowtable_lock); 475 list_for_each_entry(flowtable, &flowtables, list) 476 nf_flow_table_iterate_cleanup(flowtable, dev); 477 mutex_unlock(&flowtable_lock); 478 } 479 EXPORT_SYMBOL_GPL(nf_flow_table_cleanup); 480 481 void nf_flow_table_free(struct nf_flowtable *flow_table) 482 { 483 mutex_lock(&flowtable_lock); 484 list_del(&flow_table->list); 485 mutex_unlock(&flowtable_lock); 486 cancel_delayed_work_sync(&flow_table->gc_work); 487 nf_flow_table_iterate(flow_table, nf_flow_table_do_cleanup, NULL); 488 nf_flow_table_iterate(flow_table, nf_flow_offload_gc_step, flow_table); 489 rhashtable_destroy(&flow_table->rhashtable); 490 } 491 EXPORT_SYMBOL_GPL(nf_flow_table_free); 492 493 MODULE_LICENSE("GPL"); 494 MODULE_AUTHOR("Pablo Neira Ayuso <pablo@netfilter.org>"); 495