1 /* 2 * INET An implementation of the TCP/IP protocol suite for the LINUX 3 * operating system. INET is implemented using the BSD Socket 4 * interface as the means of communication with the user level. 5 * 6 * ROUTE - implementation of the IP router. 7 * 8 * Authors: Ross Biro 9 * Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG> 10 * Alan Cox, <gw4pts@gw4pts.ampr.org> 11 * Linus Torvalds, <Linus.Torvalds@helsinki.fi> 12 * Alexey Kuznetsov, <kuznet@ms2.inr.ac.ru> 13 * 14 * Fixes: 15 * Alan Cox : Verify area fixes. 16 * Alan Cox : cli() protects routing changes 17 * Rui Oliveira : ICMP routing table updates 18 * (rco@di.uminho.pt) Routing table insertion and update 19 * Linus Torvalds : Rewrote bits to be sensible 20 * Alan Cox : Added BSD route gw semantics 21 * Alan Cox : Super /proc >4K 22 * Alan Cox : MTU in route table 23 * Alan Cox : MSS actually. Also added the window 24 * clamper. 25 * Sam Lantinga : Fixed route matching in rt_del() 26 * Alan Cox : Routing cache support. 27 * Alan Cox : Removed compatibility cruft. 28 * Alan Cox : RTF_REJECT support. 29 * Alan Cox : TCP irtt support. 30 * Jonathan Naylor : Added Metric support. 31 * Miquel van Smoorenburg : BSD API fixes. 32 * Miquel van Smoorenburg : Metrics. 33 * Alan Cox : Use __u32 properly 34 * Alan Cox : Aligned routing errors more closely with BSD 35 * our system is still very different. 36 * Alan Cox : Faster /proc handling 37 * Alexey Kuznetsov : Massive rework to support tree based routing, 38 * routing caches and better behaviour. 39 * 40 * Olaf Erb : irtt wasn't being copied right. 41 * Bjorn Ekwall : Kerneld route support. 42 * Alan Cox : Multicast fixed (I hope) 43 * Pavel Krauz : Limited broadcast fixed 44 * Mike McLagan : Routing by source 45 * Alexey Kuznetsov : End of old history. Split to fib.c and 46 * route.c and rewritten from scratch. 47 * Andi Kleen : Load-limit warning messages. 48 * Vitaly E. Lavrov : Transparent proxy revived after year coma. 49 * Vitaly E. Lavrov : Race condition in ip_route_input_slow. 50 * Tobias Ringstrom : Uninitialized res.type in ip_route_output_slow. 51 * Vladimir V. Ivanov : IP rule info (flowid) is really useful. 52 * Marc Boucher : routing by fwmark 53 * Robert Olsson : Added rt_cache statistics 54 * Arnaldo C. Melo : Convert proc stuff to seq_file 55 * Eric Dumazet : hashed spinlocks and rt_check_expire() fixes. 56 * Ilia Sotnikov : Ignore TOS on PMTUD and Redirect 57 * Ilia Sotnikov : Removed TOS from hash calculations 58 * 59 * This program is free software; you can redistribute it and/or 60 * modify it under the terms of the GNU General Public License 61 * as published by the Free Software Foundation; either version 62 * 2 of the License, or (at your option) any later version. 63 */ 64 65 #define pr_fmt(fmt) "IPv4: " fmt 66 67 #include <linux/module.h> 68 #include <linux/uaccess.h> 69 #include <linux/bitops.h> 70 #include <linux/types.h> 71 #include <linux/kernel.h> 72 #include <linux/mm.h> 73 #include <linux/string.h> 74 #include <linux/socket.h> 75 #include <linux/sockios.h> 76 #include <linux/errno.h> 77 #include <linux/in.h> 78 #include <linux/inet.h> 79 #include <linux/netdevice.h> 80 #include <linux/proc_fs.h> 81 #include <linux/init.h> 82 #include <linux/skbuff.h> 83 #include <linux/inetdevice.h> 84 #include <linux/igmp.h> 85 #include <linux/pkt_sched.h> 86 #include <linux/mroute.h> 87 #include <linux/netfilter_ipv4.h> 88 #include <linux/random.h> 89 #include <linux/rcupdate.h> 90 #include <linux/times.h> 91 #include <linux/slab.h> 92 #include <linux/jhash.h> 93 #include <net/dst.h> 94 #include <net/dst_metadata.h> 95 #include <net/net_namespace.h> 96 #include <net/protocol.h> 97 #include <net/ip.h> 98 #include <net/route.h> 99 #include <net/inetpeer.h> 100 #include <net/sock.h> 101 #include <net/ip_fib.h> 102 #include <net/arp.h> 103 #include <net/tcp.h> 104 #include <net/icmp.h> 105 #include <net/xfrm.h> 106 #include <net/lwtunnel.h> 107 #include <net/netevent.h> 108 #include <net/rtnetlink.h> 109 #ifdef CONFIG_SYSCTL 110 #include <linux/sysctl.h> 111 #endif 112 #include <net/secure_seq.h> 113 #include <net/ip_tunnels.h> 114 #include <net/l3mdev.h> 115 116 #include "fib_lookup.h" 117 118 #define RT_FL_TOS(oldflp4) \ 119 ((oldflp4)->flowi4_tos & (IPTOS_RT_MASK | RTO_ONLINK)) 120 121 #define RT_GC_TIMEOUT (300*HZ) 122 123 static int ip_rt_max_size; 124 static int ip_rt_redirect_number __read_mostly = 9; 125 static int ip_rt_redirect_load __read_mostly = HZ / 50; 126 static int ip_rt_redirect_silence __read_mostly = ((HZ / 50) << (9 + 1)); 127 static int ip_rt_error_cost __read_mostly = HZ; 128 static int ip_rt_error_burst __read_mostly = 5 * HZ; 129 static int ip_rt_mtu_expires __read_mostly = 10 * 60 * HZ; 130 static u32 ip_rt_min_pmtu __read_mostly = 512 + 20 + 20; 131 static int ip_rt_min_advmss __read_mostly = 256; 132 133 static int ip_rt_gc_timeout __read_mostly = RT_GC_TIMEOUT; 134 135 /* 136 * Interface to generic destination cache. 137 */ 138 139 static struct dst_entry *ipv4_dst_check(struct dst_entry *dst, u32 cookie); 140 static unsigned int ipv4_default_advmss(const struct dst_entry *dst); 141 static unsigned int ipv4_mtu(const struct dst_entry *dst); 142 static struct dst_entry *ipv4_negative_advice(struct dst_entry *dst); 143 static void ipv4_link_failure(struct sk_buff *skb); 144 static void ip_rt_update_pmtu(struct dst_entry *dst, struct sock *sk, 145 struct sk_buff *skb, u32 mtu); 146 static void ip_do_redirect(struct dst_entry *dst, struct sock *sk, 147 struct sk_buff *skb); 148 static void ipv4_dst_destroy(struct dst_entry *dst); 149 150 static u32 *ipv4_cow_metrics(struct dst_entry *dst, unsigned long old) 151 { 152 WARN_ON(1); 153 return NULL; 154 } 155 156 static struct neighbour *ipv4_neigh_lookup(const struct dst_entry *dst, 157 struct sk_buff *skb, 158 const void *daddr); 159 static void ipv4_confirm_neigh(const struct dst_entry *dst, const void *daddr); 160 161 static struct dst_ops ipv4_dst_ops = { 162 .family = AF_INET, 163 .check = ipv4_dst_check, 164 .default_advmss = ipv4_default_advmss, 165 .mtu = ipv4_mtu, 166 .cow_metrics = ipv4_cow_metrics, 167 .destroy = ipv4_dst_destroy, 168 .negative_advice = ipv4_negative_advice, 169 .link_failure = ipv4_link_failure, 170 .update_pmtu = ip_rt_update_pmtu, 171 .redirect = ip_do_redirect, 172 .local_out = __ip_local_out, 173 .neigh_lookup = ipv4_neigh_lookup, 174 .confirm_neigh = ipv4_confirm_neigh, 175 }; 176 177 #define ECN_OR_COST(class) TC_PRIO_##class 178 179 const __u8 ip_tos2prio[16] = { 180 TC_PRIO_BESTEFFORT, 181 ECN_OR_COST(BESTEFFORT), 182 TC_PRIO_BESTEFFORT, 183 ECN_OR_COST(BESTEFFORT), 184 TC_PRIO_BULK, 185 ECN_OR_COST(BULK), 186 TC_PRIO_BULK, 187 ECN_OR_COST(BULK), 188 TC_PRIO_INTERACTIVE, 189 ECN_OR_COST(INTERACTIVE), 190 TC_PRIO_INTERACTIVE, 191 ECN_OR_COST(INTERACTIVE), 192 TC_PRIO_INTERACTIVE_BULK, 193 ECN_OR_COST(INTERACTIVE_BULK), 194 TC_PRIO_INTERACTIVE_BULK, 195 ECN_OR_COST(INTERACTIVE_BULK) 196 }; 197 EXPORT_SYMBOL(ip_tos2prio); 198 199 static DEFINE_PER_CPU(struct rt_cache_stat, rt_cache_stat); 200 #define RT_CACHE_STAT_INC(field) raw_cpu_inc(rt_cache_stat.field) 201 202 #ifdef CONFIG_PROC_FS 203 static void *rt_cache_seq_start(struct seq_file *seq, loff_t *pos) 204 { 205 if (*pos) 206 return NULL; 207 return SEQ_START_TOKEN; 208 } 209 210 static void *rt_cache_seq_next(struct seq_file *seq, void *v, loff_t *pos) 211 { 212 ++*pos; 213 return NULL; 214 } 215 216 static void rt_cache_seq_stop(struct seq_file *seq, void *v) 217 { 218 } 219 220 static int rt_cache_seq_show(struct seq_file *seq, void *v) 221 { 222 if (v == SEQ_START_TOKEN) 223 seq_printf(seq, "%-127s\n", 224 "Iface\tDestination\tGateway \tFlags\t\tRefCnt\tUse\t" 225 "Metric\tSource\t\tMTU\tWindow\tIRTT\tTOS\tHHRef\t" 226 "HHUptod\tSpecDst"); 227 return 0; 228 } 229 230 static const struct seq_operations rt_cache_seq_ops = { 231 .start = rt_cache_seq_start, 232 .next = rt_cache_seq_next, 233 .stop = rt_cache_seq_stop, 234 .show = rt_cache_seq_show, 235 }; 236 237 static int rt_cache_seq_open(struct inode *inode, struct file *file) 238 { 239 return seq_open(file, &rt_cache_seq_ops); 240 } 241 242 static const struct file_operations rt_cache_seq_fops = { 243 .open = rt_cache_seq_open, 244 .read = seq_read, 245 .llseek = seq_lseek, 246 .release = seq_release, 247 }; 248 249 250 static void *rt_cpu_seq_start(struct seq_file *seq, loff_t *pos) 251 { 252 int cpu; 253 254 if (*pos == 0) 255 return SEQ_START_TOKEN; 256 257 for (cpu = *pos-1; cpu < nr_cpu_ids; ++cpu) { 258 if (!cpu_possible(cpu)) 259 continue; 260 *pos = cpu+1; 261 return &per_cpu(rt_cache_stat, cpu); 262 } 263 return NULL; 264 } 265 266 static void *rt_cpu_seq_next(struct seq_file *seq, void *v, loff_t *pos) 267 { 268 int cpu; 269 270 for (cpu = *pos; cpu < nr_cpu_ids; ++cpu) { 271 if (!cpu_possible(cpu)) 272 continue; 273 *pos = cpu+1; 274 return &per_cpu(rt_cache_stat, cpu); 275 } 276 return NULL; 277 278 } 279 280 static void rt_cpu_seq_stop(struct seq_file *seq, void *v) 281 { 282 283 } 284 285 static int rt_cpu_seq_show(struct seq_file *seq, void *v) 286 { 287 struct rt_cache_stat *st = v; 288 289 if (v == SEQ_START_TOKEN) { 290 seq_printf(seq, "entries in_hit in_slow_tot in_slow_mc in_no_route in_brd in_martian_dst in_martian_src out_hit out_slow_tot out_slow_mc gc_total gc_ignored gc_goal_miss gc_dst_overflow in_hlist_search out_hlist_search\n"); 291 return 0; 292 } 293 294 seq_printf(seq,"%08x %08x %08x %08x %08x %08x %08x %08x " 295 " %08x %08x %08x %08x %08x %08x %08x %08x %08x \n", 296 dst_entries_get_slow(&ipv4_dst_ops), 297 0, /* st->in_hit */ 298 st->in_slow_tot, 299 st->in_slow_mc, 300 st->in_no_route, 301 st->in_brd, 302 st->in_martian_dst, 303 st->in_martian_src, 304 305 0, /* st->out_hit */ 306 st->out_slow_tot, 307 st->out_slow_mc, 308 309 0, /* st->gc_total */ 310 0, /* st->gc_ignored */ 311 0, /* st->gc_goal_miss */ 312 0, /* st->gc_dst_overflow */ 313 0, /* st->in_hlist_search */ 314 0 /* st->out_hlist_search */ 315 ); 316 return 0; 317 } 318 319 static const struct seq_operations rt_cpu_seq_ops = { 320 .start = rt_cpu_seq_start, 321 .next = rt_cpu_seq_next, 322 .stop = rt_cpu_seq_stop, 323 .show = rt_cpu_seq_show, 324 }; 325 326 327 static int rt_cpu_seq_open(struct inode *inode, struct file *file) 328 { 329 return seq_open(file, &rt_cpu_seq_ops); 330 } 331 332 static const struct file_operations rt_cpu_seq_fops = { 333 .open = rt_cpu_seq_open, 334 .read = seq_read, 335 .llseek = seq_lseek, 336 .release = seq_release, 337 }; 338 339 #ifdef CONFIG_IP_ROUTE_CLASSID 340 static int rt_acct_proc_show(struct seq_file *m, void *v) 341 { 342 struct ip_rt_acct *dst, *src; 343 unsigned int i, j; 344 345 dst = kcalloc(256, sizeof(struct ip_rt_acct), GFP_KERNEL); 346 if (!dst) 347 return -ENOMEM; 348 349 for_each_possible_cpu(i) { 350 src = (struct ip_rt_acct *)per_cpu_ptr(ip_rt_acct, i); 351 for (j = 0; j < 256; j++) { 352 dst[j].o_bytes += src[j].o_bytes; 353 dst[j].o_packets += src[j].o_packets; 354 dst[j].i_bytes += src[j].i_bytes; 355 dst[j].i_packets += src[j].i_packets; 356 } 357 } 358 359 seq_write(m, dst, 256 * sizeof(struct ip_rt_acct)); 360 kfree(dst); 361 return 0; 362 } 363 #endif 364 365 static int __net_init ip_rt_do_proc_init(struct net *net) 366 { 367 struct proc_dir_entry *pde; 368 369 pde = proc_create("rt_cache", 0444, net->proc_net, 370 &rt_cache_seq_fops); 371 if (!pde) 372 goto err1; 373 374 pde = proc_create("rt_cache", 0444, 375 net->proc_net_stat, &rt_cpu_seq_fops); 376 if (!pde) 377 goto err2; 378 379 #ifdef CONFIG_IP_ROUTE_CLASSID 380 pde = proc_create_single("rt_acct", 0, net->proc_net, 381 rt_acct_proc_show); 382 if (!pde) 383 goto err3; 384 #endif 385 return 0; 386 387 #ifdef CONFIG_IP_ROUTE_CLASSID 388 err3: 389 remove_proc_entry("rt_cache", net->proc_net_stat); 390 #endif 391 err2: 392 remove_proc_entry("rt_cache", net->proc_net); 393 err1: 394 return -ENOMEM; 395 } 396 397 static void __net_exit ip_rt_do_proc_exit(struct net *net) 398 { 399 remove_proc_entry("rt_cache", net->proc_net_stat); 400 remove_proc_entry("rt_cache", net->proc_net); 401 #ifdef CONFIG_IP_ROUTE_CLASSID 402 remove_proc_entry("rt_acct", net->proc_net); 403 #endif 404 } 405 406 static struct pernet_operations ip_rt_proc_ops __net_initdata = { 407 .init = ip_rt_do_proc_init, 408 .exit = ip_rt_do_proc_exit, 409 }; 410 411 static int __init ip_rt_proc_init(void) 412 { 413 return register_pernet_subsys(&ip_rt_proc_ops); 414 } 415 416 #else 417 static inline int ip_rt_proc_init(void) 418 { 419 return 0; 420 } 421 #endif /* CONFIG_PROC_FS */ 422 423 static inline bool rt_is_expired(const struct rtable *rth) 424 { 425 return rth->rt_genid != rt_genid_ipv4(dev_net(rth->dst.dev)); 426 } 427 428 void rt_cache_flush(struct net *net) 429 { 430 rt_genid_bump_ipv4(net); 431 } 432 433 static struct neighbour *ipv4_neigh_lookup(const struct dst_entry *dst, 434 struct sk_buff *skb, 435 const void *daddr) 436 { 437 struct net_device *dev = dst->dev; 438 const __be32 *pkey = daddr; 439 const struct rtable *rt; 440 struct neighbour *n; 441 442 rt = (const struct rtable *) dst; 443 if (rt->rt_gateway) 444 pkey = (const __be32 *) &rt->rt_gateway; 445 else if (skb) 446 pkey = &ip_hdr(skb)->daddr; 447 448 n = __ipv4_neigh_lookup(dev, *(__force u32 *)pkey); 449 if (n) 450 return n; 451 return neigh_create(&arp_tbl, pkey, dev); 452 } 453 454 static void ipv4_confirm_neigh(const struct dst_entry *dst, const void *daddr) 455 { 456 struct net_device *dev = dst->dev; 457 const __be32 *pkey = daddr; 458 const struct rtable *rt; 459 460 rt = (const struct rtable *)dst; 461 if (rt->rt_gateway) 462 pkey = (const __be32 *)&rt->rt_gateway; 463 else if (!daddr || 464 (rt->rt_flags & 465 (RTCF_MULTICAST | RTCF_BROADCAST | RTCF_LOCAL))) 466 return; 467 468 __ipv4_confirm_neigh(dev, *(__force u32 *)pkey); 469 } 470 471 #define IP_IDENTS_SZ 2048u 472 473 static atomic_t *ip_idents __read_mostly; 474 static u32 *ip_tstamps __read_mostly; 475 476 /* In order to protect privacy, we add a perturbation to identifiers 477 * if one generator is seldom used. This makes hard for an attacker 478 * to infer how many packets were sent between two points in time. 479 */ 480 u32 ip_idents_reserve(u32 hash, int segs) 481 { 482 u32 *p_tstamp = ip_tstamps + hash % IP_IDENTS_SZ; 483 atomic_t *p_id = ip_idents + hash % IP_IDENTS_SZ; 484 u32 old = READ_ONCE(*p_tstamp); 485 u32 now = (u32)jiffies; 486 u32 new, delta = 0; 487 488 if (old != now && cmpxchg(p_tstamp, old, now) == old) 489 delta = prandom_u32_max(now - old); 490 491 /* Do not use atomic_add_return() as it makes UBSAN unhappy */ 492 do { 493 old = (u32)atomic_read(p_id); 494 new = old + delta + segs; 495 } while (atomic_cmpxchg(p_id, old, new) != old); 496 497 return new - segs; 498 } 499 EXPORT_SYMBOL(ip_idents_reserve); 500 501 void __ip_select_ident(struct net *net, struct iphdr *iph, int segs) 502 { 503 static u32 ip_idents_hashrnd __read_mostly; 504 u32 hash, id; 505 506 net_get_random_once(&ip_idents_hashrnd, sizeof(ip_idents_hashrnd)); 507 508 hash = jhash_3words((__force u32)iph->daddr, 509 (__force u32)iph->saddr, 510 iph->protocol ^ net_hash_mix(net), 511 ip_idents_hashrnd); 512 id = ip_idents_reserve(hash, segs); 513 iph->id = htons(id); 514 } 515 EXPORT_SYMBOL(__ip_select_ident); 516 517 static void __build_flow_key(const struct net *net, struct flowi4 *fl4, 518 const struct sock *sk, 519 const struct iphdr *iph, 520 int oif, u8 tos, 521 u8 prot, u32 mark, int flow_flags) 522 { 523 if (sk) { 524 const struct inet_sock *inet = inet_sk(sk); 525 526 oif = sk->sk_bound_dev_if; 527 mark = sk->sk_mark; 528 tos = RT_CONN_FLAGS(sk); 529 prot = inet->hdrincl ? IPPROTO_RAW : sk->sk_protocol; 530 } 531 flowi4_init_output(fl4, oif, mark, tos, 532 RT_SCOPE_UNIVERSE, prot, 533 flow_flags, 534 iph->daddr, iph->saddr, 0, 0, 535 sock_net_uid(net, sk)); 536 } 537 538 static void build_skb_flow_key(struct flowi4 *fl4, const struct sk_buff *skb, 539 const struct sock *sk) 540 { 541 const struct net *net = dev_net(skb->dev); 542 const struct iphdr *iph = ip_hdr(skb); 543 int oif = skb->dev->ifindex; 544 u8 tos = RT_TOS(iph->tos); 545 u8 prot = iph->protocol; 546 u32 mark = skb->mark; 547 548 __build_flow_key(net, fl4, sk, iph, oif, tos, prot, mark, 0); 549 } 550 551 static void build_sk_flow_key(struct flowi4 *fl4, const struct sock *sk) 552 { 553 const struct inet_sock *inet = inet_sk(sk); 554 const struct ip_options_rcu *inet_opt; 555 __be32 daddr = inet->inet_daddr; 556 557 rcu_read_lock(); 558 inet_opt = rcu_dereference(inet->inet_opt); 559 if (inet_opt && inet_opt->opt.srr) 560 daddr = inet_opt->opt.faddr; 561 flowi4_init_output(fl4, sk->sk_bound_dev_if, sk->sk_mark, 562 RT_CONN_FLAGS(sk), RT_SCOPE_UNIVERSE, 563 inet->hdrincl ? IPPROTO_RAW : sk->sk_protocol, 564 inet_sk_flowi_flags(sk), 565 daddr, inet->inet_saddr, 0, 0, sk->sk_uid); 566 rcu_read_unlock(); 567 } 568 569 static void ip_rt_build_flow_key(struct flowi4 *fl4, const struct sock *sk, 570 const struct sk_buff *skb) 571 { 572 if (skb) 573 build_skb_flow_key(fl4, skb, sk); 574 else 575 build_sk_flow_key(fl4, sk); 576 } 577 578 static DEFINE_SPINLOCK(fnhe_lock); 579 580 static void fnhe_flush_routes(struct fib_nh_exception *fnhe) 581 { 582 struct rtable *rt; 583 584 rt = rcu_dereference(fnhe->fnhe_rth_input); 585 if (rt) { 586 RCU_INIT_POINTER(fnhe->fnhe_rth_input, NULL); 587 dst_dev_put(&rt->dst); 588 dst_release(&rt->dst); 589 } 590 rt = rcu_dereference(fnhe->fnhe_rth_output); 591 if (rt) { 592 RCU_INIT_POINTER(fnhe->fnhe_rth_output, NULL); 593 dst_dev_put(&rt->dst); 594 dst_release(&rt->dst); 595 } 596 } 597 598 static struct fib_nh_exception *fnhe_oldest(struct fnhe_hash_bucket *hash) 599 { 600 struct fib_nh_exception *fnhe, *oldest; 601 602 oldest = rcu_dereference(hash->chain); 603 for (fnhe = rcu_dereference(oldest->fnhe_next); fnhe; 604 fnhe = rcu_dereference(fnhe->fnhe_next)) { 605 if (time_before(fnhe->fnhe_stamp, oldest->fnhe_stamp)) 606 oldest = fnhe; 607 } 608 fnhe_flush_routes(oldest); 609 return oldest; 610 } 611 612 static inline u32 fnhe_hashfun(__be32 daddr) 613 { 614 static u32 fnhe_hashrnd __read_mostly; 615 u32 hval; 616 617 net_get_random_once(&fnhe_hashrnd, sizeof(fnhe_hashrnd)); 618 hval = jhash_1word((__force u32) daddr, fnhe_hashrnd); 619 return hash_32(hval, FNHE_HASH_SHIFT); 620 } 621 622 static void fill_route_from_fnhe(struct rtable *rt, struct fib_nh_exception *fnhe) 623 { 624 rt->rt_pmtu = fnhe->fnhe_pmtu; 625 rt->rt_mtu_locked = fnhe->fnhe_mtu_locked; 626 rt->dst.expires = fnhe->fnhe_expires; 627 628 if (fnhe->fnhe_gw) { 629 rt->rt_flags |= RTCF_REDIRECTED; 630 rt->rt_gateway = fnhe->fnhe_gw; 631 rt->rt_uses_gateway = 1; 632 } 633 } 634 635 static void update_or_create_fnhe(struct fib_nh *nh, __be32 daddr, __be32 gw, 636 u32 pmtu, bool lock, unsigned long expires) 637 { 638 struct fnhe_hash_bucket *hash; 639 struct fib_nh_exception *fnhe; 640 struct rtable *rt; 641 u32 genid, hval; 642 unsigned int i; 643 int depth; 644 645 genid = fnhe_genid(dev_net(nh->nh_dev)); 646 hval = fnhe_hashfun(daddr); 647 648 spin_lock_bh(&fnhe_lock); 649 650 hash = rcu_dereference(nh->nh_exceptions); 651 if (!hash) { 652 hash = kcalloc(FNHE_HASH_SIZE, sizeof(*hash), GFP_ATOMIC); 653 if (!hash) 654 goto out_unlock; 655 rcu_assign_pointer(nh->nh_exceptions, hash); 656 } 657 658 hash += hval; 659 660 depth = 0; 661 for (fnhe = rcu_dereference(hash->chain); fnhe; 662 fnhe = rcu_dereference(fnhe->fnhe_next)) { 663 if (fnhe->fnhe_daddr == daddr) 664 break; 665 depth++; 666 } 667 668 if (fnhe) { 669 if (fnhe->fnhe_genid != genid) 670 fnhe->fnhe_genid = genid; 671 if (gw) 672 fnhe->fnhe_gw = gw; 673 if (pmtu) { 674 fnhe->fnhe_pmtu = pmtu; 675 fnhe->fnhe_mtu_locked = lock; 676 } 677 fnhe->fnhe_expires = max(1UL, expires); 678 /* Update all cached dsts too */ 679 rt = rcu_dereference(fnhe->fnhe_rth_input); 680 if (rt) 681 fill_route_from_fnhe(rt, fnhe); 682 rt = rcu_dereference(fnhe->fnhe_rth_output); 683 if (rt) 684 fill_route_from_fnhe(rt, fnhe); 685 } else { 686 if (depth > FNHE_RECLAIM_DEPTH) 687 fnhe = fnhe_oldest(hash); 688 else { 689 fnhe = kzalloc(sizeof(*fnhe), GFP_ATOMIC); 690 if (!fnhe) 691 goto out_unlock; 692 693 fnhe->fnhe_next = hash->chain; 694 rcu_assign_pointer(hash->chain, fnhe); 695 } 696 fnhe->fnhe_genid = genid; 697 fnhe->fnhe_daddr = daddr; 698 fnhe->fnhe_gw = gw; 699 fnhe->fnhe_pmtu = pmtu; 700 fnhe->fnhe_mtu_locked = lock; 701 fnhe->fnhe_expires = max(1UL, expires); 702 703 /* Exception created; mark the cached routes for the nexthop 704 * stale, so anyone caching it rechecks if this exception 705 * applies to them. 706 */ 707 rt = rcu_dereference(nh->nh_rth_input); 708 if (rt) 709 rt->dst.obsolete = DST_OBSOLETE_KILL; 710 711 for_each_possible_cpu(i) { 712 struct rtable __rcu **prt; 713 prt = per_cpu_ptr(nh->nh_pcpu_rth_output, i); 714 rt = rcu_dereference(*prt); 715 if (rt) 716 rt->dst.obsolete = DST_OBSOLETE_KILL; 717 } 718 } 719 720 fnhe->fnhe_stamp = jiffies; 721 722 out_unlock: 723 spin_unlock_bh(&fnhe_lock); 724 } 725 726 static void __ip_do_redirect(struct rtable *rt, struct sk_buff *skb, struct flowi4 *fl4, 727 bool kill_route) 728 { 729 __be32 new_gw = icmp_hdr(skb)->un.gateway; 730 __be32 old_gw = ip_hdr(skb)->saddr; 731 struct net_device *dev = skb->dev; 732 struct in_device *in_dev; 733 struct fib_result res; 734 struct neighbour *n; 735 struct net *net; 736 737 switch (icmp_hdr(skb)->code & 7) { 738 case ICMP_REDIR_NET: 739 case ICMP_REDIR_NETTOS: 740 case ICMP_REDIR_HOST: 741 case ICMP_REDIR_HOSTTOS: 742 break; 743 744 default: 745 return; 746 } 747 748 if (rt->rt_gateway != old_gw) 749 return; 750 751 in_dev = __in_dev_get_rcu(dev); 752 if (!in_dev) 753 return; 754 755 net = dev_net(dev); 756 if (new_gw == old_gw || !IN_DEV_RX_REDIRECTS(in_dev) || 757 ipv4_is_multicast(new_gw) || ipv4_is_lbcast(new_gw) || 758 ipv4_is_zeronet(new_gw)) 759 goto reject_redirect; 760 761 if (!IN_DEV_SHARED_MEDIA(in_dev)) { 762 if (!inet_addr_onlink(in_dev, new_gw, old_gw)) 763 goto reject_redirect; 764 if (IN_DEV_SEC_REDIRECTS(in_dev) && ip_fib_check_default(new_gw, dev)) 765 goto reject_redirect; 766 } else { 767 if (inet_addr_type(net, new_gw) != RTN_UNICAST) 768 goto reject_redirect; 769 } 770 771 n = __ipv4_neigh_lookup(rt->dst.dev, new_gw); 772 if (!n) 773 n = neigh_create(&arp_tbl, &new_gw, rt->dst.dev); 774 if (!IS_ERR(n)) { 775 if (!(n->nud_state & NUD_VALID)) { 776 neigh_event_send(n, NULL); 777 } else { 778 if (fib_lookup(net, fl4, &res, 0) == 0) { 779 struct fib_nh *nh = &FIB_RES_NH(res); 780 781 update_or_create_fnhe(nh, fl4->daddr, new_gw, 782 0, false, 783 jiffies + ip_rt_gc_timeout); 784 } 785 if (kill_route) 786 rt->dst.obsolete = DST_OBSOLETE_KILL; 787 call_netevent_notifiers(NETEVENT_NEIGH_UPDATE, n); 788 } 789 neigh_release(n); 790 } 791 return; 792 793 reject_redirect: 794 #ifdef CONFIG_IP_ROUTE_VERBOSE 795 if (IN_DEV_LOG_MARTIANS(in_dev)) { 796 const struct iphdr *iph = (const struct iphdr *) skb->data; 797 __be32 daddr = iph->daddr; 798 __be32 saddr = iph->saddr; 799 800 net_info_ratelimited("Redirect from %pI4 on %s about %pI4 ignored\n" 801 " Advised path = %pI4 -> %pI4\n", 802 &old_gw, dev->name, &new_gw, 803 &saddr, &daddr); 804 } 805 #endif 806 ; 807 } 808 809 static void ip_do_redirect(struct dst_entry *dst, struct sock *sk, struct sk_buff *skb) 810 { 811 struct rtable *rt; 812 struct flowi4 fl4; 813 const struct iphdr *iph = (const struct iphdr *) skb->data; 814 struct net *net = dev_net(skb->dev); 815 int oif = skb->dev->ifindex; 816 u8 tos = RT_TOS(iph->tos); 817 u8 prot = iph->protocol; 818 u32 mark = skb->mark; 819 820 rt = (struct rtable *) dst; 821 822 __build_flow_key(net, &fl4, sk, iph, oif, tos, prot, mark, 0); 823 __ip_do_redirect(rt, skb, &fl4, true); 824 } 825 826 static struct dst_entry *ipv4_negative_advice(struct dst_entry *dst) 827 { 828 struct rtable *rt = (struct rtable *)dst; 829 struct dst_entry *ret = dst; 830 831 if (rt) { 832 if (dst->obsolete > 0) { 833 ip_rt_put(rt); 834 ret = NULL; 835 } else if ((rt->rt_flags & RTCF_REDIRECTED) || 836 rt->dst.expires) { 837 ip_rt_put(rt); 838 ret = NULL; 839 } 840 } 841 return ret; 842 } 843 844 /* 845 * Algorithm: 846 * 1. The first ip_rt_redirect_number redirects are sent 847 * with exponential backoff, then we stop sending them at all, 848 * assuming that the host ignores our redirects. 849 * 2. If we did not see packets requiring redirects 850 * during ip_rt_redirect_silence, we assume that the host 851 * forgot redirected route and start to send redirects again. 852 * 853 * This algorithm is much cheaper and more intelligent than dumb load limiting 854 * in icmp.c. 855 * 856 * NOTE. Do not forget to inhibit load limiting for redirects (redundant) 857 * and "frag. need" (breaks PMTU discovery) in icmp.c. 858 */ 859 860 void ip_rt_send_redirect(struct sk_buff *skb) 861 { 862 struct rtable *rt = skb_rtable(skb); 863 struct in_device *in_dev; 864 struct inet_peer *peer; 865 struct net *net; 866 int log_martians; 867 int vif; 868 869 rcu_read_lock(); 870 in_dev = __in_dev_get_rcu(rt->dst.dev); 871 if (!in_dev || !IN_DEV_TX_REDIRECTS(in_dev)) { 872 rcu_read_unlock(); 873 return; 874 } 875 log_martians = IN_DEV_LOG_MARTIANS(in_dev); 876 vif = l3mdev_master_ifindex_rcu(rt->dst.dev); 877 rcu_read_unlock(); 878 879 net = dev_net(rt->dst.dev); 880 peer = inet_getpeer_v4(net->ipv4.peers, ip_hdr(skb)->saddr, vif, 1); 881 if (!peer) { 882 icmp_send(skb, ICMP_REDIRECT, ICMP_REDIR_HOST, 883 rt_nexthop(rt, ip_hdr(skb)->daddr)); 884 return; 885 } 886 887 /* No redirected packets during ip_rt_redirect_silence; 888 * reset the algorithm. 889 */ 890 if (time_after(jiffies, peer->rate_last + ip_rt_redirect_silence)) { 891 peer->rate_tokens = 0; 892 peer->n_redirects = 0; 893 } 894 895 /* Too many ignored redirects; do not send anything 896 * set dst.rate_last to the last seen redirected packet. 897 */ 898 if (peer->n_redirects >= ip_rt_redirect_number) { 899 peer->rate_last = jiffies; 900 goto out_put_peer; 901 } 902 903 /* Check for load limit; set rate_last to the latest sent 904 * redirect. 905 */ 906 if (peer->rate_tokens == 0 || 907 time_after(jiffies, 908 (peer->rate_last + 909 (ip_rt_redirect_load << peer->rate_tokens)))) { 910 __be32 gw = rt_nexthop(rt, ip_hdr(skb)->daddr); 911 912 icmp_send(skb, ICMP_REDIRECT, ICMP_REDIR_HOST, gw); 913 peer->rate_last = jiffies; 914 ++peer->rate_tokens; 915 ++peer->n_redirects; 916 #ifdef CONFIG_IP_ROUTE_VERBOSE 917 if (log_martians && 918 peer->rate_tokens == ip_rt_redirect_number) 919 net_warn_ratelimited("host %pI4/if%d ignores redirects for %pI4 to %pI4\n", 920 &ip_hdr(skb)->saddr, inet_iif(skb), 921 &ip_hdr(skb)->daddr, &gw); 922 #endif 923 } 924 out_put_peer: 925 inet_putpeer(peer); 926 } 927 928 static int ip_error(struct sk_buff *skb) 929 { 930 struct rtable *rt = skb_rtable(skb); 931 struct net_device *dev = skb->dev; 932 struct in_device *in_dev; 933 struct inet_peer *peer; 934 unsigned long now; 935 struct net *net; 936 bool send; 937 int code; 938 939 if (netif_is_l3_master(skb->dev)) { 940 dev = __dev_get_by_index(dev_net(skb->dev), IPCB(skb)->iif); 941 if (!dev) 942 goto out; 943 } 944 945 in_dev = __in_dev_get_rcu(dev); 946 947 /* IP on this device is disabled. */ 948 if (!in_dev) 949 goto out; 950 951 net = dev_net(rt->dst.dev); 952 if (!IN_DEV_FORWARD(in_dev)) { 953 switch (rt->dst.error) { 954 case EHOSTUNREACH: 955 __IP_INC_STATS(net, IPSTATS_MIB_INADDRERRORS); 956 break; 957 958 case ENETUNREACH: 959 __IP_INC_STATS(net, IPSTATS_MIB_INNOROUTES); 960 break; 961 } 962 goto out; 963 } 964 965 switch (rt->dst.error) { 966 case EINVAL: 967 default: 968 goto out; 969 case EHOSTUNREACH: 970 code = ICMP_HOST_UNREACH; 971 break; 972 case ENETUNREACH: 973 code = ICMP_NET_UNREACH; 974 __IP_INC_STATS(net, IPSTATS_MIB_INNOROUTES); 975 break; 976 case EACCES: 977 code = ICMP_PKT_FILTERED; 978 break; 979 } 980 981 peer = inet_getpeer_v4(net->ipv4.peers, ip_hdr(skb)->saddr, 982 l3mdev_master_ifindex(skb->dev), 1); 983 984 send = true; 985 if (peer) { 986 now = jiffies; 987 peer->rate_tokens += now - peer->rate_last; 988 if (peer->rate_tokens > ip_rt_error_burst) 989 peer->rate_tokens = ip_rt_error_burst; 990 peer->rate_last = now; 991 if (peer->rate_tokens >= ip_rt_error_cost) 992 peer->rate_tokens -= ip_rt_error_cost; 993 else 994 send = false; 995 inet_putpeer(peer); 996 } 997 if (send) 998 icmp_send(skb, ICMP_DEST_UNREACH, code, 0); 999 1000 out: kfree_skb(skb); 1001 return 0; 1002 } 1003 1004 static void __ip_rt_update_pmtu(struct rtable *rt, struct flowi4 *fl4, u32 mtu) 1005 { 1006 struct dst_entry *dst = &rt->dst; 1007 u32 old_mtu = ipv4_mtu(dst); 1008 struct fib_result res; 1009 bool lock = false; 1010 1011 if (ip_mtu_locked(dst)) 1012 return; 1013 1014 if (old_mtu < mtu) 1015 return; 1016 1017 if (mtu < ip_rt_min_pmtu) { 1018 lock = true; 1019 mtu = min(old_mtu, ip_rt_min_pmtu); 1020 } 1021 1022 if (rt->rt_pmtu == mtu && !lock && 1023 time_before(jiffies, dst->expires - ip_rt_mtu_expires / 2)) 1024 return; 1025 1026 rcu_read_lock(); 1027 if (fib_lookup(dev_net(dst->dev), fl4, &res, 0) == 0) { 1028 struct fib_nh *nh = &FIB_RES_NH(res); 1029 1030 update_or_create_fnhe(nh, fl4->daddr, 0, mtu, lock, 1031 jiffies + ip_rt_mtu_expires); 1032 } 1033 rcu_read_unlock(); 1034 } 1035 1036 static void ip_rt_update_pmtu(struct dst_entry *dst, struct sock *sk, 1037 struct sk_buff *skb, u32 mtu) 1038 { 1039 struct rtable *rt = (struct rtable *) dst; 1040 struct flowi4 fl4; 1041 1042 ip_rt_build_flow_key(&fl4, sk, skb); 1043 __ip_rt_update_pmtu(rt, &fl4, mtu); 1044 } 1045 1046 void ipv4_update_pmtu(struct sk_buff *skb, struct net *net, u32 mtu, 1047 int oif, u8 protocol) 1048 { 1049 const struct iphdr *iph = (const struct iphdr *) skb->data; 1050 struct flowi4 fl4; 1051 struct rtable *rt; 1052 u32 mark = IP4_REPLY_MARK(net, skb->mark); 1053 1054 __build_flow_key(net, &fl4, NULL, iph, oif, 1055 RT_TOS(iph->tos), protocol, mark, 0); 1056 rt = __ip_route_output_key(net, &fl4); 1057 if (!IS_ERR(rt)) { 1058 __ip_rt_update_pmtu(rt, &fl4, mtu); 1059 ip_rt_put(rt); 1060 } 1061 } 1062 EXPORT_SYMBOL_GPL(ipv4_update_pmtu); 1063 1064 static void __ipv4_sk_update_pmtu(struct sk_buff *skb, struct sock *sk, u32 mtu) 1065 { 1066 const struct iphdr *iph = (const struct iphdr *) skb->data; 1067 struct flowi4 fl4; 1068 struct rtable *rt; 1069 1070 __build_flow_key(sock_net(sk), &fl4, sk, iph, 0, 0, 0, 0, 0); 1071 1072 if (!fl4.flowi4_mark) 1073 fl4.flowi4_mark = IP4_REPLY_MARK(sock_net(sk), skb->mark); 1074 1075 rt = __ip_route_output_key(sock_net(sk), &fl4); 1076 if (!IS_ERR(rt)) { 1077 __ip_rt_update_pmtu(rt, &fl4, mtu); 1078 ip_rt_put(rt); 1079 } 1080 } 1081 1082 void ipv4_sk_update_pmtu(struct sk_buff *skb, struct sock *sk, u32 mtu) 1083 { 1084 const struct iphdr *iph = (const struct iphdr *) skb->data; 1085 struct flowi4 fl4; 1086 struct rtable *rt; 1087 struct dst_entry *odst = NULL; 1088 bool new = false; 1089 struct net *net = sock_net(sk); 1090 1091 bh_lock_sock(sk); 1092 1093 if (!ip_sk_accept_pmtu(sk)) 1094 goto out; 1095 1096 odst = sk_dst_get(sk); 1097 1098 if (sock_owned_by_user(sk) || !odst) { 1099 __ipv4_sk_update_pmtu(skb, sk, mtu); 1100 goto out; 1101 } 1102 1103 __build_flow_key(net, &fl4, sk, iph, 0, 0, 0, 0, 0); 1104 1105 rt = (struct rtable *)odst; 1106 if (odst->obsolete && !odst->ops->check(odst, 0)) { 1107 rt = ip_route_output_flow(sock_net(sk), &fl4, sk); 1108 if (IS_ERR(rt)) 1109 goto out; 1110 1111 new = true; 1112 } 1113 1114 __ip_rt_update_pmtu((struct rtable *) xfrm_dst_path(&rt->dst), &fl4, mtu); 1115 1116 if (!dst_check(&rt->dst, 0)) { 1117 if (new) 1118 dst_release(&rt->dst); 1119 1120 rt = ip_route_output_flow(sock_net(sk), &fl4, sk); 1121 if (IS_ERR(rt)) 1122 goto out; 1123 1124 new = true; 1125 } 1126 1127 if (new) 1128 sk_dst_set(sk, &rt->dst); 1129 1130 out: 1131 bh_unlock_sock(sk); 1132 dst_release(odst); 1133 } 1134 EXPORT_SYMBOL_GPL(ipv4_sk_update_pmtu); 1135 1136 void ipv4_redirect(struct sk_buff *skb, struct net *net, 1137 int oif, u8 protocol) 1138 { 1139 const struct iphdr *iph = (const struct iphdr *) skb->data; 1140 struct flowi4 fl4; 1141 struct rtable *rt; 1142 1143 __build_flow_key(net, &fl4, NULL, iph, oif, 1144 RT_TOS(iph->tos), protocol, 0, 0); 1145 rt = __ip_route_output_key(net, &fl4); 1146 if (!IS_ERR(rt)) { 1147 __ip_do_redirect(rt, skb, &fl4, false); 1148 ip_rt_put(rt); 1149 } 1150 } 1151 EXPORT_SYMBOL_GPL(ipv4_redirect); 1152 1153 void ipv4_sk_redirect(struct sk_buff *skb, struct sock *sk) 1154 { 1155 const struct iphdr *iph = (const struct iphdr *) skb->data; 1156 struct flowi4 fl4; 1157 struct rtable *rt; 1158 struct net *net = sock_net(sk); 1159 1160 __build_flow_key(net, &fl4, sk, iph, 0, 0, 0, 0, 0); 1161 rt = __ip_route_output_key(net, &fl4); 1162 if (!IS_ERR(rt)) { 1163 __ip_do_redirect(rt, skb, &fl4, false); 1164 ip_rt_put(rt); 1165 } 1166 } 1167 EXPORT_SYMBOL_GPL(ipv4_sk_redirect); 1168 1169 static struct dst_entry *ipv4_dst_check(struct dst_entry *dst, u32 cookie) 1170 { 1171 struct rtable *rt = (struct rtable *) dst; 1172 1173 /* All IPV4 dsts are created with ->obsolete set to the value 1174 * DST_OBSOLETE_FORCE_CHK which forces validation calls down 1175 * into this function always. 1176 * 1177 * When a PMTU/redirect information update invalidates a route, 1178 * this is indicated by setting obsolete to DST_OBSOLETE_KILL or 1179 * DST_OBSOLETE_DEAD by dst_free(). 1180 */ 1181 if (dst->obsolete != DST_OBSOLETE_FORCE_CHK || rt_is_expired(rt)) 1182 return NULL; 1183 return dst; 1184 } 1185 1186 static void ipv4_link_failure(struct sk_buff *skb) 1187 { 1188 struct rtable *rt; 1189 1190 icmp_send(skb, ICMP_DEST_UNREACH, ICMP_HOST_UNREACH, 0); 1191 1192 rt = skb_rtable(skb); 1193 if (rt) 1194 dst_set_expires(&rt->dst, 0); 1195 } 1196 1197 static int ip_rt_bug(struct net *net, struct sock *sk, struct sk_buff *skb) 1198 { 1199 pr_debug("%s: %pI4 -> %pI4, %s\n", 1200 __func__, &ip_hdr(skb)->saddr, &ip_hdr(skb)->daddr, 1201 skb->dev ? skb->dev->name : "?"); 1202 kfree_skb(skb); 1203 WARN_ON(1); 1204 return 0; 1205 } 1206 1207 /* 1208 We do not cache source address of outgoing interface, 1209 because it is used only by IP RR, TS and SRR options, 1210 so that it out of fast path. 1211 1212 BTW remember: "addr" is allowed to be not aligned 1213 in IP options! 1214 */ 1215 1216 void ip_rt_get_source(u8 *addr, struct sk_buff *skb, struct rtable *rt) 1217 { 1218 __be32 src; 1219 1220 if (rt_is_output_route(rt)) 1221 src = ip_hdr(skb)->saddr; 1222 else { 1223 struct fib_result res; 1224 struct iphdr *iph = ip_hdr(skb); 1225 struct flowi4 fl4 = { 1226 .daddr = iph->daddr, 1227 .saddr = iph->saddr, 1228 .flowi4_tos = RT_TOS(iph->tos), 1229 .flowi4_oif = rt->dst.dev->ifindex, 1230 .flowi4_iif = skb->dev->ifindex, 1231 .flowi4_mark = skb->mark, 1232 }; 1233 1234 rcu_read_lock(); 1235 if (fib_lookup(dev_net(rt->dst.dev), &fl4, &res, 0) == 0) 1236 src = FIB_RES_PREFSRC(dev_net(rt->dst.dev), res); 1237 else 1238 src = inet_select_addr(rt->dst.dev, 1239 rt_nexthop(rt, iph->daddr), 1240 RT_SCOPE_UNIVERSE); 1241 rcu_read_unlock(); 1242 } 1243 memcpy(addr, &src, 4); 1244 } 1245 1246 #ifdef CONFIG_IP_ROUTE_CLASSID 1247 static void set_class_tag(struct rtable *rt, u32 tag) 1248 { 1249 if (!(rt->dst.tclassid & 0xFFFF)) 1250 rt->dst.tclassid |= tag & 0xFFFF; 1251 if (!(rt->dst.tclassid & 0xFFFF0000)) 1252 rt->dst.tclassid |= tag & 0xFFFF0000; 1253 } 1254 #endif 1255 1256 static unsigned int ipv4_default_advmss(const struct dst_entry *dst) 1257 { 1258 unsigned int header_size = sizeof(struct tcphdr) + sizeof(struct iphdr); 1259 unsigned int advmss = max_t(unsigned int, ipv4_mtu(dst) - header_size, 1260 ip_rt_min_advmss); 1261 1262 return min(advmss, IPV4_MAX_PMTU - header_size); 1263 } 1264 1265 static unsigned int ipv4_mtu(const struct dst_entry *dst) 1266 { 1267 const struct rtable *rt = (const struct rtable *) dst; 1268 unsigned int mtu = rt->rt_pmtu; 1269 1270 if (!mtu || time_after_eq(jiffies, rt->dst.expires)) 1271 mtu = dst_metric_raw(dst, RTAX_MTU); 1272 1273 if (mtu) 1274 return mtu; 1275 1276 mtu = READ_ONCE(dst->dev->mtu); 1277 1278 if (unlikely(ip_mtu_locked(dst))) { 1279 if (rt->rt_uses_gateway && mtu > 576) 1280 mtu = 576; 1281 } 1282 1283 mtu = min_t(unsigned int, mtu, IP_MAX_MTU); 1284 1285 return mtu - lwtunnel_headroom(dst->lwtstate, mtu); 1286 } 1287 1288 static void ip_del_fnhe(struct fib_nh *nh, __be32 daddr) 1289 { 1290 struct fnhe_hash_bucket *hash; 1291 struct fib_nh_exception *fnhe, __rcu **fnhe_p; 1292 u32 hval = fnhe_hashfun(daddr); 1293 1294 spin_lock_bh(&fnhe_lock); 1295 1296 hash = rcu_dereference_protected(nh->nh_exceptions, 1297 lockdep_is_held(&fnhe_lock)); 1298 hash += hval; 1299 1300 fnhe_p = &hash->chain; 1301 fnhe = rcu_dereference_protected(*fnhe_p, lockdep_is_held(&fnhe_lock)); 1302 while (fnhe) { 1303 if (fnhe->fnhe_daddr == daddr) { 1304 rcu_assign_pointer(*fnhe_p, rcu_dereference_protected( 1305 fnhe->fnhe_next, lockdep_is_held(&fnhe_lock))); 1306 fnhe_flush_routes(fnhe); 1307 kfree_rcu(fnhe, rcu); 1308 break; 1309 } 1310 fnhe_p = &fnhe->fnhe_next; 1311 fnhe = rcu_dereference_protected(fnhe->fnhe_next, 1312 lockdep_is_held(&fnhe_lock)); 1313 } 1314 1315 spin_unlock_bh(&fnhe_lock); 1316 } 1317 1318 static struct fib_nh_exception *find_exception(struct fib_nh *nh, __be32 daddr) 1319 { 1320 struct fnhe_hash_bucket *hash = rcu_dereference(nh->nh_exceptions); 1321 struct fib_nh_exception *fnhe; 1322 u32 hval; 1323 1324 if (!hash) 1325 return NULL; 1326 1327 hval = fnhe_hashfun(daddr); 1328 1329 for (fnhe = rcu_dereference(hash[hval].chain); fnhe; 1330 fnhe = rcu_dereference(fnhe->fnhe_next)) { 1331 if (fnhe->fnhe_daddr == daddr) { 1332 if (fnhe->fnhe_expires && 1333 time_after(jiffies, fnhe->fnhe_expires)) { 1334 ip_del_fnhe(nh, daddr); 1335 break; 1336 } 1337 return fnhe; 1338 } 1339 } 1340 return NULL; 1341 } 1342 1343 /* MTU selection: 1344 * 1. mtu on route is locked - use it 1345 * 2. mtu from nexthop exception 1346 * 3. mtu from egress device 1347 */ 1348 1349 u32 ip_mtu_from_fib_result(struct fib_result *res, __be32 daddr) 1350 { 1351 struct fib_info *fi = res->fi; 1352 struct fib_nh *nh = &fi->fib_nh[res->nh_sel]; 1353 struct net_device *dev = nh->nh_dev; 1354 u32 mtu = 0; 1355 1356 if (dev_net(dev)->ipv4.sysctl_ip_fwd_use_pmtu || 1357 fi->fib_metrics->metrics[RTAX_LOCK - 1] & (1 << RTAX_MTU)) 1358 mtu = fi->fib_mtu; 1359 1360 if (likely(!mtu)) { 1361 struct fib_nh_exception *fnhe; 1362 1363 fnhe = find_exception(nh, daddr); 1364 if (fnhe && !time_after_eq(jiffies, fnhe->fnhe_expires)) 1365 mtu = fnhe->fnhe_pmtu; 1366 } 1367 1368 if (likely(!mtu)) 1369 mtu = min(READ_ONCE(dev->mtu), IP_MAX_MTU); 1370 1371 return mtu - lwtunnel_headroom(nh->nh_lwtstate, mtu); 1372 } 1373 1374 static bool rt_bind_exception(struct rtable *rt, struct fib_nh_exception *fnhe, 1375 __be32 daddr, const bool do_cache) 1376 { 1377 bool ret = false; 1378 1379 spin_lock_bh(&fnhe_lock); 1380 1381 if (daddr == fnhe->fnhe_daddr) { 1382 struct rtable __rcu **porig; 1383 struct rtable *orig; 1384 int genid = fnhe_genid(dev_net(rt->dst.dev)); 1385 1386 if (rt_is_input_route(rt)) 1387 porig = &fnhe->fnhe_rth_input; 1388 else 1389 porig = &fnhe->fnhe_rth_output; 1390 orig = rcu_dereference(*porig); 1391 1392 if (fnhe->fnhe_genid != genid) { 1393 fnhe->fnhe_genid = genid; 1394 fnhe->fnhe_gw = 0; 1395 fnhe->fnhe_pmtu = 0; 1396 fnhe->fnhe_expires = 0; 1397 fnhe->fnhe_mtu_locked = false; 1398 fnhe_flush_routes(fnhe); 1399 orig = NULL; 1400 } 1401 fill_route_from_fnhe(rt, fnhe); 1402 if (!rt->rt_gateway) 1403 rt->rt_gateway = daddr; 1404 1405 if (do_cache) { 1406 dst_hold(&rt->dst); 1407 rcu_assign_pointer(*porig, rt); 1408 if (orig) { 1409 dst_dev_put(&orig->dst); 1410 dst_release(&orig->dst); 1411 } 1412 ret = true; 1413 } 1414 1415 fnhe->fnhe_stamp = jiffies; 1416 } 1417 spin_unlock_bh(&fnhe_lock); 1418 1419 return ret; 1420 } 1421 1422 static bool rt_cache_route(struct fib_nh *nh, struct rtable *rt) 1423 { 1424 struct rtable *orig, *prev, **p; 1425 bool ret = true; 1426 1427 if (rt_is_input_route(rt)) { 1428 p = (struct rtable **)&nh->nh_rth_input; 1429 } else { 1430 p = (struct rtable **)raw_cpu_ptr(nh->nh_pcpu_rth_output); 1431 } 1432 orig = *p; 1433 1434 /* hold dst before doing cmpxchg() to avoid race condition 1435 * on this dst 1436 */ 1437 dst_hold(&rt->dst); 1438 prev = cmpxchg(p, orig, rt); 1439 if (prev == orig) { 1440 if (orig) { 1441 dst_dev_put(&orig->dst); 1442 dst_release(&orig->dst); 1443 } 1444 } else { 1445 dst_release(&rt->dst); 1446 ret = false; 1447 } 1448 1449 return ret; 1450 } 1451 1452 struct uncached_list { 1453 spinlock_t lock; 1454 struct list_head head; 1455 }; 1456 1457 static DEFINE_PER_CPU_ALIGNED(struct uncached_list, rt_uncached_list); 1458 1459 void rt_add_uncached_list(struct rtable *rt) 1460 { 1461 struct uncached_list *ul = raw_cpu_ptr(&rt_uncached_list); 1462 1463 rt->rt_uncached_list = ul; 1464 1465 spin_lock_bh(&ul->lock); 1466 list_add_tail(&rt->rt_uncached, &ul->head); 1467 spin_unlock_bh(&ul->lock); 1468 } 1469 1470 void rt_del_uncached_list(struct rtable *rt) 1471 { 1472 if (!list_empty(&rt->rt_uncached)) { 1473 struct uncached_list *ul = rt->rt_uncached_list; 1474 1475 spin_lock_bh(&ul->lock); 1476 list_del(&rt->rt_uncached); 1477 spin_unlock_bh(&ul->lock); 1478 } 1479 } 1480 1481 static void ipv4_dst_destroy(struct dst_entry *dst) 1482 { 1483 struct rtable *rt = (struct rtable *)dst; 1484 1485 ip_dst_metrics_put(dst); 1486 rt_del_uncached_list(rt); 1487 } 1488 1489 void rt_flush_dev(struct net_device *dev) 1490 { 1491 struct net *net = dev_net(dev); 1492 struct rtable *rt; 1493 int cpu; 1494 1495 for_each_possible_cpu(cpu) { 1496 struct uncached_list *ul = &per_cpu(rt_uncached_list, cpu); 1497 1498 spin_lock_bh(&ul->lock); 1499 list_for_each_entry(rt, &ul->head, rt_uncached) { 1500 if (rt->dst.dev != dev) 1501 continue; 1502 rt->dst.dev = net->loopback_dev; 1503 dev_hold(rt->dst.dev); 1504 dev_put(dev); 1505 } 1506 spin_unlock_bh(&ul->lock); 1507 } 1508 } 1509 1510 static bool rt_cache_valid(const struct rtable *rt) 1511 { 1512 return rt && 1513 rt->dst.obsolete == DST_OBSOLETE_FORCE_CHK && 1514 !rt_is_expired(rt); 1515 } 1516 1517 static void rt_set_nexthop(struct rtable *rt, __be32 daddr, 1518 const struct fib_result *res, 1519 struct fib_nh_exception *fnhe, 1520 struct fib_info *fi, u16 type, u32 itag, 1521 const bool do_cache) 1522 { 1523 bool cached = false; 1524 1525 if (fi) { 1526 struct fib_nh *nh = &FIB_RES_NH(*res); 1527 1528 if (nh->nh_gw && nh->nh_scope == RT_SCOPE_LINK) { 1529 rt->rt_gateway = nh->nh_gw; 1530 rt->rt_uses_gateway = 1; 1531 } 1532 ip_dst_init_metrics(&rt->dst, fi->fib_metrics); 1533 1534 #ifdef CONFIG_IP_ROUTE_CLASSID 1535 rt->dst.tclassid = nh->nh_tclassid; 1536 #endif 1537 rt->dst.lwtstate = lwtstate_get(nh->nh_lwtstate); 1538 if (unlikely(fnhe)) 1539 cached = rt_bind_exception(rt, fnhe, daddr, do_cache); 1540 else if (do_cache) 1541 cached = rt_cache_route(nh, rt); 1542 if (unlikely(!cached)) { 1543 /* Routes we intend to cache in nexthop exception or 1544 * FIB nexthop have the DST_NOCACHE bit clear. 1545 * However, if we are unsuccessful at storing this 1546 * route into the cache we really need to set it. 1547 */ 1548 if (!rt->rt_gateway) 1549 rt->rt_gateway = daddr; 1550 rt_add_uncached_list(rt); 1551 } 1552 } else 1553 rt_add_uncached_list(rt); 1554 1555 #ifdef CONFIG_IP_ROUTE_CLASSID 1556 #ifdef CONFIG_IP_MULTIPLE_TABLES 1557 set_class_tag(rt, res->tclassid); 1558 #endif 1559 set_class_tag(rt, itag); 1560 #endif 1561 } 1562 1563 struct rtable *rt_dst_alloc(struct net_device *dev, 1564 unsigned int flags, u16 type, 1565 bool nopolicy, bool noxfrm, bool will_cache) 1566 { 1567 struct rtable *rt; 1568 1569 rt = dst_alloc(&ipv4_dst_ops, dev, 1, DST_OBSOLETE_FORCE_CHK, 1570 (will_cache ? 0 : DST_HOST) | 1571 (nopolicy ? DST_NOPOLICY : 0) | 1572 (noxfrm ? DST_NOXFRM : 0)); 1573 1574 if (rt) { 1575 rt->rt_genid = rt_genid_ipv4(dev_net(dev)); 1576 rt->rt_flags = flags; 1577 rt->rt_type = type; 1578 rt->rt_is_input = 0; 1579 rt->rt_iif = 0; 1580 rt->rt_pmtu = 0; 1581 rt->rt_mtu_locked = 0; 1582 rt->rt_gateway = 0; 1583 rt->rt_uses_gateway = 0; 1584 INIT_LIST_HEAD(&rt->rt_uncached); 1585 1586 rt->dst.output = ip_output; 1587 if (flags & RTCF_LOCAL) 1588 rt->dst.input = ip_local_deliver; 1589 } 1590 1591 return rt; 1592 } 1593 EXPORT_SYMBOL(rt_dst_alloc); 1594 1595 /* called in rcu_read_lock() section */ 1596 int ip_mc_validate_source(struct sk_buff *skb, __be32 daddr, __be32 saddr, 1597 u8 tos, struct net_device *dev, 1598 struct in_device *in_dev, u32 *itag) 1599 { 1600 int err; 1601 1602 /* Primary sanity checks. */ 1603 if (!in_dev) 1604 return -EINVAL; 1605 1606 if (ipv4_is_multicast(saddr) || ipv4_is_lbcast(saddr) || 1607 skb->protocol != htons(ETH_P_IP)) 1608 return -EINVAL; 1609 1610 if (ipv4_is_loopback(saddr) && !IN_DEV_ROUTE_LOCALNET(in_dev)) 1611 return -EINVAL; 1612 1613 if (ipv4_is_zeronet(saddr)) { 1614 if (!ipv4_is_local_multicast(daddr)) 1615 return -EINVAL; 1616 } else { 1617 err = fib_validate_source(skb, saddr, 0, tos, 0, dev, 1618 in_dev, itag); 1619 if (err < 0) 1620 return err; 1621 } 1622 return 0; 1623 } 1624 1625 /* called in rcu_read_lock() section */ 1626 static int ip_route_input_mc(struct sk_buff *skb, __be32 daddr, __be32 saddr, 1627 u8 tos, struct net_device *dev, int our) 1628 { 1629 struct in_device *in_dev = __in_dev_get_rcu(dev); 1630 unsigned int flags = RTCF_MULTICAST; 1631 struct rtable *rth; 1632 u32 itag = 0; 1633 int err; 1634 1635 err = ip_mc_validate_source(skb, daddr, saddr, tos, dev, in_dev, &itag); 1636 if (err) 1637 return err; 1638 1639 if (our) 1640 flags |= RTCF_LOCAL; 1641 1642 rth = rt_dst_alloc(dev_net(dev)->loopback_dev, flags, RTN_MULTICAST, 1643 IN_DEV_CONF_GET(in_dev, NOPOLICY), false, false); 1644 if (!rth) 1645 return -ENOBUFS; 1646 1647 #ifdef CONFIG_IP_ROUTE_CLASSID 1648 rth->dst.tclassid = itag; 1649 #endif 1650 rth->dst.output = ip_rt_bug; 1651 rth->rt_is_input= 1; 1652 1653 #ifdef CONFIG_IP_MROUTE 1654 if (!ipv4_is_local_multicast(daddr) && IN_DEV_MFORWARD(in_dev)) 1655 rth->dst.input = ip_mr_input; 1656 #endif 1657 RT_CACHE_STAT_INC(in_slow_mc); 1658 1659 skb_dst_set(skb, &rth->dst); 1660 return 0; 1661 } 1662 1663 1664 static void ip_handle_martian_source(struct net_device *dev, 1665 struct in_device *in_dev, 1666 struct sk_buff *skb, 1667 __be32 daddr, 1668 __be32 saddr) 1669 { 1670 RT_CACHE_STAT_INC(in_martian_src); 1671 #ifdef CONFIG_IP_ROUTE_VERBOSE 1672 if (IN_DEV_LOG_MARTIANS(in_dev) && net_ratelimit()) { 1673 /* 1674 * RFC1812 recommendation, if source is martian, 1675 * the only hint is MAC header. 1676 */ 1677 pr_warn("martian source %pI4 from %pI4, on dev %s\n", 1678 &daddr, &saddr, dev->name); 1679 if (dev->hard_header_len && skb_mac_header_was_set(skb)) { 1680 print_hex_dump(KERN_WARNING, "ll header: ", 1681 DUMP_PREFIX_OFFSET, 16, 1, 1682 skb_mac_header(skb), 1683 dev->hard_header_len, false); 1684 } 1685 } 1686 #endif 1687 } 1688 1689 /* called in rcu_read_lock() section */ 1690 static int __mkroute_input(struct sk_buff *skb, 1691 const struct fib_result *res, 1692 struct in_device *in_dev, 1693 __be32 daddr, __be32 saddr, u32 tos) 1694 { 1695 struct fib_nh_exception *fnhe; 1696 struct rtable *rth; 1697 int err; 1698 struct in_device *out_dev; 1699 bool do_cache; 1700 u32 itag = 0; 1701 1702 /* get a working reference to the output device */ 1703 out_dev = __in_dev_get_rcu(FIB_RES_DEV(*res)); 1704 if (!out_dev) { 1705 net_crit_ratelimited("Bug in ip_route_input_slow(). Please report.\n"); 1706 return -EINVAL; 1707 } 1708 1709 err = fib_validate_source(skb, saddr, daddr, tos, FIB_RES_OIF(*res), 1710 in_dev->dev, in_dev, &itag); 1711 if (err < 0) { 1712 ip_handle_martian_source(in_dev->dev, in_dev, skb, daddr, 1713 saddr); 1714 1715 goto cleanup; 1716 } 1717 1718 do_cache = res->fi && !itag; 1719 if (out_dev == in_dev && err && IN_DEV_TX_REDIRECTS(out_dev) && 1720 skb->protocol == htons(ETH_P_IP) && 1721 (IN_DEV_SHARED_MEDIA(out_dev) || 1722 inet_addr_onlink(out_dev, saddr, FIB_RES_GW(*res)))) 1723 IPCB(skb)->flags |= IPSKB_DOREDIRECT; 1724 1725 if (skb->protocol != htons(ETH_P_IP)) { 1726 /* Not IP (i.e. ARP). Do not create route, if it is 1727 * invalid for proxy arp. DNAT routes are always valid. 1728 * 1729 * Proxy arp feature have been extended to allow, ARP 1730 * replies back to the same interface, to support 1731 * Private VLAN switch technologies. See arp.c. 1732 */ 1733 if (out_dev == in_dev && 1734 IN_DEV_PROXY_ARP_PVLAN(in_dev) == 0) { 1735 err = -EINVAL; 1736 goto cleanup; 1737 } 1738 } 1739 1740 fnhe = find_exception(&FIB_RES_NH(*res), daddr); 1741 if (do_cache) { 1742 if (fnhe) 1743 rth = rcu_dereference(fnhe->fnhe_rth_input); 1744 else 1745 rth = rcu_dereference(FIB_RES_NH(*res).nh_rth_input); 1746 if (rt_cache_valid(rth)) { 1747 skb_dst_set_noref(skb, &rth->dst); 1748 goto out; 1749 } 1750 } 1751 1752 rth = rt_dst_alloc(out_dev->dev, 0, res->type, 1753 IN_DEV_CONF_GET(in_dev, NOPOLICY), 1754 IN_DEV_CONF_GET(out_dev, NOXFRM), do_cache); 1755 if (!rth) { 1756 err = -ENOBUFS; 1757 goto cleanup; 1758 } 1759 1760 rth->rt_is_input = 1; 1761 RT_CACHE_STAT_INC(in_slow_tot); 1762 1763 rth->dst.input = ip_forward; 1764 1765 rt_set_nexthop(rth, daddr, res, fnhe, res->fi, res->type, itag, 1766 do_cache); 1767 lwtunnel_set_redirect(&rth->dst); 1768 skb_dst_set(skb, &rth->dst); 1769 out: 1770 err = 0; 1771 cleanup: 1772 return err; 1773 } 1774 1775 #ifdef CONFIG_IP_ROUTE_MULTIPATH 1776 /* To make ICMP packets follow the right flow, the multipath hash is 1777 * calculated from the inner IP addresses. 1778 */ 1779 static void ip_multipath_l3_keys(const struct sk_buff *skb, 1780 struct flow_keys *hash_keys) 1781 { 1782 const struct iphdr *outer_iph = ip_hdr(skb); 1783 const struct iphdr *key_iph = outer_iph; 1784 const struct iphdr *inner_iph; 1785 const struct icmphdr *icmph; 1786 struct iphdr _inner_iph; 1787 struct icmphdr _icmph; 1788 1789 if (likely(outer_iph->protocol != IPPROTO_ICMP)) 1790 goto out; 1791 1792 if (unlikely((outer_iph->frag_off & htons(IP_OFFSET)) != 0)) 1793 goto out; 1794 1795 icmph = skb_header_pointer(skb, outer_iph->ihl * 4, sizeof(_icmph), 1796 &_icmph); 1797 if (!icmph) 1798 goto out; 1799 1800 if (icmph->type != ICMP_DEST_UNREACH && 1801 icmph->type != ICMP_REDIRECT && 1802 icmph->type != ICMP_TIME_EXCEEDED && 1803 icmph->type != ICMP_PARAMETERPROB) 1804 goto out; 1805 1806 inner_iph = skb_header_pointer(skb, 1807 outer_iph->ihl * 4 + sizeof(_icmph), 1808 sizeof(_inner_iph), &_inner_iph); 1809 if (!inner_iph) 1810 goto out; 1811 1812 key_iph = inner_iph; 1813 out: 1814 hash_keys->addrs.v4addrs.src = key_iph->saddr; 1815 hash_keys->addrs.v4addrs.dst = key_iph->daddr; 1816 } 1817 1818 /* if skb is set it will be used and fl4 can be NULL */ 1819 int fib_multipath_hash(const struct net *net, const struct flowi4 *fl4, 1820 const struct sk_buff *skb, struct flow_keys *flkeys) 1821 { 1822 struct flow_keys hash_keys; 1823 u32 mhash; 1824 1825 switch (net->ipv4.sysctl_fib_multipath_hash_policy) { 1826 case 0: 1827 memset(&hash_keys, 0, sizeof(hash_keys)); 1828 hash_keys.control.addr_type = FLOW_DISSECTOR_KEY_IPV4_ADDRS; 1829 if (skb) { 1830 ip_multipath_l3_keys(skb, &hash_keys); 1831 } else { 1832 hash_keys.addrs.v4addrs.src = fl4->saddr; 1833 hash_keys.addrs.v4addrs.dst = fl4->daddr; 1834 } 1835 break; 1836 case 1: 1837 /* skb is currently provided only when forwarding */ 1838 if (skb) { 1839 unsigned int flag = FLOW_DISSECTOR_F_STOP_AT_ENCAP; 1840 struct flow_keys keys; 1841 1842 /* short-circuit if we already have L4 hash present */ 1843 if (skb->l4_hash) 1844 return skb_get_hash_raw(skb) >> 1; 1845 1846 memset(&hash_keys, 0, sizeof(hash_keys)); 1847 1848 if (!flkeys) { 1849 skb_flow_dissect_flow_keys(skb, &keys, flag); 1850 flkeys = &keys; 1851 } 1852 1853 hash_keys.control.addr_type = FLOW_DISSECTOR_KEY_IPV4_ADDRS; 1854 hash_keys.addrs.v4addrs.src = flkeys->addrs.v4addrs.src; 1855 hash_keys.addrs.v4addrs.dst = flkeys->addrs.v4addrs.dst; 1856 hash_keys.ports.src = flkeys->ports.src; 1857 hash_keys.ports.dst = flkeys->ports.dst; 1858 hash_keys.basic.ip_proto = flkeys->basic.ip_proto; 1859 } else { 1860 memset(&hash_keys, 0, sizeof(hash_keys)); 1861 hash_keys.control.addr_type = FLOW_DISSECTOR_KEY_IPV4_ADDRS; 1862 hash_keys.addrs.v4addrs.src = fl4->saddr; 1863 hash_keys.addrs.v4addrs.dst = fl4->daddr; 1864 hash_keys.ports.src = fl4->fl4_sport; 1865 hash_keys.ports.dst = fl4->fl4_dport; 1866 hash_keys.basic.ip_proto = fl4->flowi4_proto; 1867 } 1868 break; 1869 } 1870 mhash = flow_hash_from_keys(&hash_keys); 1871 1872 return mhash >> 1; 1873 } 1874 #endif /* CONFIG_IP_ROUTE_MULTIPATH */ 1875 1876 static int ip_mkroute_input(struct sk_buff *skb, 1877 struct fib_result *res, 1878 struct in_device *in_dev, 1879 __be32 daddr, __be32 saddr, u32 tos, 1880 struct flow_keys *hkeys) 1881 { 1882 #ifdef CONFIG_IP_ROUTE_MULTIPATH 1883 if (res->fi && res->fi->fib_nhs > 1) { 1884 int h = fib_multipath_hash(res->fi->fib_net, NULL, skb, hkeys); 1885 1886 fib_select_multipath(res, h); 1887 } 1888 #endif 1889 1890 /* create a routing cache entry */ 1891 return __mkroute_input(skb, res, in_dev, daddr, saddr, tos); 1892 } 1893 1894 /* 1895 * NOTE. We drop all the packets that has local source 1896 * addresses, because every properly looped back packet 1897 * must have correct destination already attached by output routine. 1898 * 1899 * Such approach solves two big problems: 1900 * 1. Not simplex devices are handled properly. 1901 * 2. IP spoofing attempts are filtered with 100% of guarantee. 1902 * called with rcu_read_lock() 1903 */ 1904 1905 static int ip_route_input_slow(struct sk_buff *skb, __be32 daddr, __be32 saddr, 1906 u8 tos, struct net_device *dev, 1907 struct fib_result *res) 1908 { 1909 struct in_device *in_dev = __in_dev_get_rcu(dev); 1910 struct flow_keys *flkeys = NULL, _flkeys; 1911 struct net *net = dev_net(dev); 1912 struct ip_tunnel_info *tun_info; 1913 int err = -EINVAL; 1914 unsigned int flags = 0; 1915 u32 itag = 0; 1916 struct rtable *rth; 1917 struct flowi4 fl4; 1918 bool do_cache; 1919 1920 /* IP on this device is disabled. */ 1921 1922 if (!in_dev) 1923 goto out; 1924 1925 /* Check for the most weird martians, which can be not detected 1926 by fib_lookup. 1927 */ 1928 1929 tun_info = skb_tunnel_info(skb); 1930 if (tun_info && !(tun_info->mode & IP_TUNNEL_INFO_TX)) 1931 fl4.flowi4_tun_key.tun_id = tun_info->key.tun_id; 1932 else 1933 fl4.flowi4_tun_key.tun_id = 0; 1934 skb_dst_drop(skb); 1935 1936 if (ipv4_is_multicast(saddr) || ipv4_is_lbcast(saddr)) 1937 goto martian_source; 1938 1939 res->fi = NULL; 1940 res->table = NULL; 1941 if (ipv4_is_lbcast(daddr) || (saddr == 0 && daddr == 0)) 1942 goto brd_input; 1943 1944 /* Accept zero addresses only to limited broadcast; 1945 * I even do not know to fix it or not. Waiting for complains :-) 1946 */ 1947 if (ipv4_is_zeronet(saddr)) 1948 goto martian_source; 1949 1950 if (ipv4_is_zeronet(daddr)) 1951 goto martian_destination; 1952 1953 /* Following code try to avoid calling IN_DEV_NET_ROUTE_LOCALNET(), 1954 * and call it once if daddr or/and saddr are loopback addresses 1955 */ 1956 if (ipv4_is_loopback(daddr)) { 1957 if (!IN_DEV_NET_ROUTE_LOCALNET(in_dev, net)) 1958 goto martian_destination; 1959 } else if (ipv4_is_loopback(saddr)) { 1960 if (!IN_DEV_NET_ROUTE_LOCALNET(in_dev, net)) 1961 goto martian_source; 1962 } 1963 1964 /* 1965 * Now we are ready to route packet. 1966 */ 1967 fl4.flowi4_oif = 0; 1968 fl4.flowi4_iif = dev->ifindex; 1969 fl4.flowi4_mark = skb->mark; 1970 fl4.flowi4_tos = tos; 1971 fl4.flowi4_scope = RT_SCOPE_UNIVERSE; 1972 fl4.flowi4_flags = 0; 1973 fl4.daddr = daddr; 1974 fl4.saddr = saddr; 1975 fl4.flowi4_uid = sock_net_uid(net, NULL); 1976 1977 if (fib4_rules_early_flow_dissect(net, skb, &fl4, &_flkeys)) { 1978 flkeys = &_flkeys; 1979 } else { 1980 fl4.flowi4_proto = 0; 1981 fl4.fl4_sport = 0; 1982 fl4.fl4_dport = 0; 1983 } 1984 1985 err = fib_lookup(net, &fl4, res, 0); 1986 if (err != 0) { 1987 if (!IN_DEV_FORWARD(in_dev)) 1988 err = -EHOSTUNREACH; 1989 goto no_route; 1990 } 1991 1992 if (res->type == RTN_BROADCAST) { 1993 if (IN_DEV_BFORWARD(in_dev)) 1994 goto make_route; 1995 goto brd_input; 1996 } 1997 1998 if (res->type == RTN_LOCAL) { 1999 err = fib_validate_source(skb, saddr, daddr, tos, 2000 0, dev, in_dev, &itag); 2001 if (err < 0) 2002 goto martian_source; 2003 goto local_input; 2004 } 2005 2006 if (!IN_DEV_FORWARD(in_dev)) { 2007 err = -EHOSTUNREACH; 2008 goto no_route; 2009 } 2010 if (res->type != RTN_UNICAST) 2011 goto martian_destination; 2012 2013 make_route: 2014 err = ip_mkroute_input(skb, res, in_dev, daddr, saddr, tos, flkeys); 2015 out: return err; 2016 2017 brd_input: 2018 if (skb->protocol != htons(ETH_P_IP)) 2019 goto e_inval; 2020 2021 if (!ipv4_is_zeronet(saddr)) { 2022 err = fib_validate_source(skb, saddr, 0, tos, 0, dev, 2023 in_dev, &itag); 2024 if (err < 0) 2025 goto martian_source; 2026 } 2027 flags |= RTCF_BROADCAST; 2028 res->type = RTN_BROADCAST; 2029 RT_CACHE_STAT_INC(in_brd); 2030 2031 local_input: 2032 do_cache = false; 2033 if (res->fi) { 2034 if (!itag) { 2035 rth = rcu_dereference(FIB_RES_NH(*res).nh_rth_input); 2036 if (rt_cache_valid(rth)) { 2037 skb_dst_set_noref(skb, &rth->dst); 2038 err = 0; 2039 goto out; 2040 } 2041 do_cache = true; 2042 } 2043 } 2044 2045 rth = rt_dst_alloc(l3mdev_master_dev_rcu(dev) ? : net->loopback_dev, 2046 flags | RTCF_LOCAL, res->type, 2047 IN_DEV_CONF_GET(in_dev, NOPOLICY), false, do_cache); 2048 if (!rth) 2049 goto e_nobufs; 2050 2051 rth->dst.output= ip_rt_bug; 2052 #ifdef CONFIG_IP_ROUTE_CLASSID 2053 rth->dst.tclassid = itag; 2054 #endif 2055 rth->rt_is_input = 1; 2056 2057 RT_CACHE_STAT_INC(in_slow_tot); 2058 if (res->type == RTN_UNREACHABLE) { 2059 rth->dst.input= ip_error; 2060 rth->dst.error= -err; 2061 rth->rt_flags &= ~RTCF_LOCAL; 2062 } 2063 2064 if (do_cache) { 2065 struct fib_nh *nh = &FIB_RES_NH(*res); 2066 2067 rth->dst.lwtstate = lwtstate_get(nh->nh_lwtstate); 2068 if (lwtunnel_input_redirect(rth->dst.lwtstate)) { 2069 WARN_ON(rth->dst.input == lwtunnel_input); 2070 rth->dst.lwtstate->orig_input = rth->dst.input; 2071 rth->dst.input = lwtunnel_input; 2072 } 2073 2074 if (unlikely(!rt_cache_route(nh, rth))) 2075 rt_add_uncached_list(rth); 2076 } 2077 skb_dst_set(skb, &rth->dst); 2078 err = 0; 2079 goto out; 2080 2081 no_route: 2082 RT_CACHE_STAT_INC(in_no_route); 2083 res->type = RTN_UNREACHABLE; 2084 res->fi = NULL; 2085 res->table = NULL; 2086 goto local_input; 2087 2088 /* 2089 * Do not cache martian addresses: they should be logged (RFC1812) 2090 */ 2091 martian_destination: 2092 RT_CACHE_STAT_INC(in_martian_dst); 2093 #ifdef CONFIG_IP_ROUTE_VERBOSE 2094 if (IN_DEV_LOG_MARTIANS(in_dev)) 2095 net_warn_ratelimited("martian destination %pI4 from %pI4, dev %s\n", 2096 &daddr, &saddr, dev->name); 2097 #endif 2098 2099 e_inval: 2100 err = -EINVAL; 2101 goto out; 2102 2103 e_nobufs: 2104 err = -ENOBUFS; 2105 goto out; 2106 2107 martian_source: 2108 ip_handle_martian_source(dev, in_dev, skb, daddr, saddr); 2109 goto out; 2110 } 2111 2112 int ip_route_input_noref(struct sk_buff *skb, __be32 daddr, __be32 saddr, 2113 u8 tos, struct net_device *dev) 2114 { 2115 struct fib_result res; 2116 int err; 2117 2118 tos &= IPTOS_RT_MASK; 2119 rcu_read_lock(); 2120 err = ip_route_input_rcu(skb, daddr, saddr, tos, dev, &res); 2121 rcu_read_unlock(); 2122 2123 return err; 2124 } 2125 EXPORT_SYMBOL(ip_route_input_noref); 2126 2127 /* called with rcu_read_lock held */ 2128 int ip_route_input_rcu(struct sk_buff *skb, __be32 daddr, __be32 saddr, 2129 u8 tos, struct net_device *dev, struct fib_result *res) 2130 { 2131 /* Multicast recognition logic is moved from route cache to here. 2132 The problem was that too many Ethernet cards have broken/missing 2133 hardware multicast filters :-( As result the host on multicasting 2134 network acquires a lot of useless route cache entries, sort of 2135 SDR messages from all the world. Now we try to get rid of them. 2136 Really, provided software IP multicast filter is organized 2137 reasonably (at least, hashed), it does not result in a slowdown 2138 comparing with route cache reject entries. 2139 Note, that multicast routers are not affected, because 2140 route cache entry is created eventually. 2141 */ 2142 if (ipv4_is_multicast(daddr)) { 2143 struct in_device *in_dev = __in_dev_get_rcu(dev); 2144 int our = 0; 2145 int err = -EINVAL; 2146 2147 if (in_dev) 2148 our = ip_check_mc_rcu(in_dev, daddr, saddr, 2149 ip_hdr(skb)->protocol); 2150 2151 /* check l3 master if no match yet */ 2152 if ((!in_dev || !our) && netif_is_l3_slave(dev)) { 2153 struct in_device *l3_in_dev; 2154 2155 l3_in_dev = __in_dev_get_rcu(skb->dev); 2156 if (l3_in_dev) 2157 our = ip_check_mc_rcu(l3_in_dev, daddr, saddr, 2158 ip_hdr(skb)->protocol); 2159 } 2160 2161 if (our 2162 #ifdef CONFIG_IP_MROUTE 2163 || 2164 (!ipv4_is_local_multicast(daddr) && 2165 IN_DEV_MFORWARD(in_dev)) 2166 #endif 2167 ) { 2168 err = ip_route_input_mc(skb, daddr, saddr, 2169 tos, dev, our); 2170 } 2171 return err; 2172 } 2173 2174 return ip_route_input_slow(skb, daddr, saddr, tos, dev, res); 2175 } 2176 2177 /* called with rcu_read_lock() */ 2178 static struct rtable *__mkroute_output(const struct fib_result *res, 2179 const struct flowi4 *fl4, int orig_oif, 2180 struct net_device *dev_out, 2181 unsigned int flags) 2182 { 2183 struct fib_info *fi = res->fi; 2184 struct fib_nh_exception *fnhe; 2185 struct in_device *in_dev; 2186 u16 type = res->type; 2187 struct rtable *rth; 2188 bool do_cache; 2189 2190 in_dev = __in_dev_get_rcu(dev_out); 2191 if (!in_dev) 2192 return ERR_PTR(-EINVAL); 2193 2194 if (likely(!IN_DEV_ROUTE_LOCALNET(in_dev))) 2195 if (ipv4_is_loopback(fl4->saddr) && 2196 !(dev_out->flags & IFF_LOOPBACK) && 2197 !netif_is_l3_master(dev_out)) 2198 return ERR_PTR(-EINVAL); 2199 2200 if (ipv4_is_lbcast(fl4->daddr)) 2201 type = RTN_BROADCAST; 2202 else if (ipv4_is_multicast(fl4->daddr)) 2203 type = RTN_MULTICAST; 2204 else if (ipv4_is_zeronet(fl4->daddr)) 2205 return ERR_PTR(-EINVAL); 2206 2207 if (dev_out->flags & IFF_LOOPBACK) 2208 flags |= RTCF_LOCAL; 2209 2210 do_cache = true; 2211 if (type == RTN_BROADCAST) { 2212 flags |= RTCF_BROADCAST | RTCF_LOCAL; 2213 fi = NULL; 2214 } else if (type == RTN_MULTICAST) { 2215 flags |= RTCF_MULTICAST | RTCF_LOCAL; 2216 if (!ip_check_mc_rcu(in_dev, fl4->daddr, fl4->saddr, 2217 fl4->flowi4_proto)) 2218 flags &= ~RTCF_LOCAL; 2219 else 2220 do_cache = false; 2221 /* If multicast route do not exist use 2222 * default one, but do not gateway in this case. 2223 * Yes, it is hack. 2224 */ 2225 if (fi && res->prefixlen < 4) 2226 fi = NULL; 2227 } else if ((type == RTN_LOCAL) && (orig_oif != 0) && 2228 (orig_oif != dev_out->ifindex)) { 2229 /* For local routes that require a particular output interface 2230 * we do not want to cache the result. Caching the result 2231 * causes incorrect behaviour when there are multiple source 2232 * addresses on the interface, the end result being that if the 2233 * intended recipient is waiting on that interface for the 2234 * packet he won't receive it because it will be delivered on 2235 * the loopback interface and the IP_PKTINFO ipi_ifindex will 2236 * be set to the loopback interface as well. 2237 */ 2238 do_cache = false; 2239 } 2240 2241 fnhe = NULL; 2242 do_cache &= fi != NULL; 2243 if (fi) { 2244 struct rtable __rcu **prth; 2245 struct fib_nh *nh = &FIB_RES_NH(*res); 2246 2247 fnhe = find_exception(nh, fl4->daddr); 2248 if (!do_cache) 2249 goto add; 2250 if (fnhe) { 2251 prth = &fnhe->fnhe_rth_output; 2252 } else { 2253 if (unlikely(fl4->flowi4_flags & 2254 FLOWI_FLAG_KNOWN_NH && 2255 !(nh->nh_gw && 2256 nh->nh_scope == RT_SCOPE_LINK))) { 2257 do_cache = false; 2258 goto add; 2259 } 2260 prth = raw_cpu_ptr(nh->nh_pcpu_rth_output); 2261 } 2262 rth = rcu_dereference(*prth); 2263 if (rt_cache_valid(rth) && dst_hold_safe(&rth->dst)) 2264 return rth; 2265 } 2266 2267 add: 2268 rth = rt_dst_alloc(dev_out, flags, type, 2269 IN_DEV_CONF_GET(in_dev, NOPOLICY), 2270 IN_DEV_CONF_GET(in_dev, NOXFRM), 2271 do_cache); 2272 if (!rth) 2273 return ERR_PTR(-ENOBUFS); 2274 2275 rth->rt_iif = orig_oif; 2276 2277 RT_CACHE_STAT_INC(out_slow_tot); 2278 2279 if (flags & (RTCF_BROADCAST | RTCF_MULTICAST)) { 2280 if (flags & RTCF_LOCAL && 2281 !(dev_out->flags & IFF_LOOPBACK)) { 2282 rth->dst.output = ip_mc_output; 2283 RT_CACHE_STAT_INC(out_slow_mc); 2284 } 2285 #ifdef CONFIG_IP_MROUTE 2286 if (type == RTN_MULTICAST) { 2287 if (IN_DEV_MFORWARD(in_dev) && 2288 !ipv4_is_local_multicast(fl4->daddr)) { 2289 rth->dst.input = ip_mr_input; 2290 rth->dst.output = ip_mc_output; 2291 } 2292 } 2293 #endif 2294 } 2295 2296 rt_set_nexthop(rth, fl4->daddr, res, fnhe, fi, type, 0, do_cache); 2297 lwtunnel_set_redirect(&rth->dst); 2298 2299 return rth; 2300 } 2301 2302 /* 2303 * Major route resolver routine. 2304 */ 2305 2306 struct rtable *ip_route_output_key_hash(struct net *net, struct flowi4 *fl4, 2307 const struct sk_buff *skb) 2308 { 2309 __u8 tos = RT_FL_TOS(fl4); 2310 struct fib_result res = { 2311 .type = RTN_UNSPEC, 2312 .fi = NULL, 2313 .table = NULL, 2314 .tclassid = 0, 2315 }; 2316 struct rtable *rth; 2317 2318 fl4->flowi4_iif = LOOPBACK_IFINDEX; 2319 fl4->flowi4_tos = tos & IPTOS_RT_MASK; 2320 fl4->flowi4_scope = ((tos & RTO_ONLINK) ? 2321 RT_SCOPE_LINK : RT_SCOPE_UNIVERSE); 2322 2323 rcu_read_lock(); 2324 rth = ip_route_output_key_hash_rcu(net, fl4, &res, skb); 2325 rcu_read_unlock(); 2326 2327 return rth; 2328 } 2329 EXPORT_SYMBOL_GPL(ip_route_output_key_hash); 2330 2331 struct rtable *ip_route_output_key_hash_rcu(struct net *net, struct flowi4 *fl4, 2332 struct fib_result *res, 2333 const struct sk_buff *skb) 2334 { 2335 struct net_device *dev_out = NULL; 2336 int orig_oif = fl4->flowi4_oif; 2337 unsigned int flags = 0; 2338 struct rtable *rth; 2339 int err = -ENETUNREACH; 2340 2341 if (fl4->saddr) { 2342 rth = ERR_PTR(-EINVAL); 2343 if (ipv4_is_multicast(fl4->saddr) || 2344 ipv4_is_lbcast(fl4->saddr) || 2345 ipv4_is_zeronet(fl4->saddr)) 2346 goto out; 2347 2348 /* I removed check for oif == dev_out->oif here. 2349 It was wrong for two reasons: 2350 1. ip_dev_find(net, saddr) can return wrong iface, if saddr 2351 is assigned to multiple interfaces. 2352 2. Moreover, we are allowed to send packets with saddr 2353 of another iface. --ANK 2354 */ 2355 2356 if (fl4->flowi4_oif == 0 && 2357 (ipv4_is_multicast(fl4->daddr) || 2358 ipv4_is_lbcast(fl4->daddr))) { 2359 /* It is equivalent to inet_addr_type(saddr) == RTN_LOCAL */ 2360 dev_out = __ip_dev_find(net, fl4->saddr, false); 2361 if (!dev_out) 2362 goto out; 2363 2364 /* Special hack: user can direct multicasts 2365 and limited broadcast via necessary interface 2366 without fiddling with IP_MULTICAST_IF or IP_PKTINFO. 2367 This hack is not just for fun, it allows 2368 vic,vat and friends to work. 2369 They bind socket to loopback, set ttl to zero 2370 and expect that it will work. 2371 From the viewpoint of routing cache they are broken, 2372 because we are not allowed to build multicast path 2373 with loopback source addr (look, routing cache 2374 cannot know, that ttl is zero, so that packet 2375 will not leave this host and route is valid). 2376 Luckily, this hack is good workaround. 2377 */ 2378 2379 fl4->flowi4_oif = dev_out->ifindex; 2380 goto make_route; 2381 } 2382 2383 if (!(fl4->flowi4_flags & FLOWI_FLAG_ANYSRC)) { 2384 /* It is equivalent to inet_addr_type(saddr) == RTN_LOCAL */ 2385 if (!__ip_dev_find(net, fl4->saddr, false)) 2386 goto out; 2387 } 2388 } 2389 2390 2391 if (fl4->flowi4_oif) { 2392 dev_out = dev_get_by_index_rcu(net, fl4->flowi4_oif); 2393 rth = ERR_PTR(-ENODEV); 2394 if (!dev_out) 2395 goto out; 2396 2397 /* RACE: Check return value of inet_select_addr instead. */ 2398 if (!(dev_out->flags & IFF_UP) || !__in_dev_get_rcu(dev_out)) { 2399 rth = ERR_PTR(-ENETUNREACH); 2400 goto out; 2401 } 2402 if (ipv4_is_local_multicast(fl4->daddr) || 2403 ipv4_is_lbcast(fl4->daddr) || 2404 fl4->flowi4_proto == IPPROTO_IGMP) { 2405 if (!fl4->saddr) 2406 fl4->saddr = inet_select_addr(dev_out, 0, 2407 RT_SCOPE_LINK); 2408 goto make_route; 2409 } 2410 if (!fl4->saddr) { 2411 if (ipv4_is_multicast(fl4->daddr)) 2412 fl4->saddr = inet_select_addr(dev_out, 0, 2413 fl4->flowi4_scope); 2414 else if (!fl4->daddr) 2415 fl4->saddr = inet_select_addr(dev_out, 0, 2416 RT_SCOPE_HOST); 2417 } 2418 } 2419 2420 if (!fl4->daddr) { 2421 fl4->daddr = fl4->saddr; 2422 if (!fl4->daddr) 2423 fl4->daddr = fl4->saddr = htonl(INADDR_LOOPBACK); 2424 dev_out = net->loopback_dev; 2425 fl4->flowi4_oif = LOOPBACK_IFINDEX; 2426 res->type = RTN_LOCAL; 2427 flags |= RTCF_LOCAL; 2428 goto make_route; 2429 } 2430 2431 err = fib_lookup(net, fl4, res, 0); 2432 if (err) { 2433 res->fi = NULL; 2434 res->table = NULL; 2435 if (fl4->flowi4_oif && 2436 (ipv4_is_multicast(fl4->daddr) || 2437 !netif_index_is_l3_master(net, fl4->flowi4_oif))) { 2438 /* Apparently, routing tables are wrong. Assume, 2439 that the destination is on link. 2440 2441 WHY? DW. 2442 Because we are allowed to send to iface 2443 even if it has NO routes and NO assigned 2444 addresses. When oif is specified, routing 2445 tables are looked up with only one purpose: 2446 to catch if destination is gatewayed, rather than 2447 direct. Moreover, if MSG_DONTROUTE is set, 2448 we send packet, ignoring both routing tables 2449 and ifaddr state. --ANK 2450 2451 2452 We could make it even if oif is unknown, 2453 likely IPv6, but we do not. 2454 */ 2455 2456 if (fl4->saddr == 0) 2457 fl4->saddr = inet_select_addr(dev_out, 0, 2458 RT_SCOPE_LINK); 2459 res->type = RTN_UNICAST; 2460 goto make_route; 2461 } 2462 rth = ERR_PTR(err); 2463 goto out; 2464 } 2465 2466 if (res->type == RTN_LOCAL) { 2467 if (!fl4->saddr) { 2468 if (res->fi->fib_prefsrc) 2469 fl4->saddr = res->fi->fib_prefsrc; 2470 else 2471 fl4->saddr = fl4->daddr; 2472 } 2473 2474 /* L3 master device is the loopback for that domain */ 2475 dev_out = l3mdev_master_dev_rcu(FIB_RES_DEV(*res)) ? : 2476 net->loopback_dev; 2477 2478 /* make sure orig_oif points to fib result device even 2479 * though packet rx/tx happens over loopback or l3mdev 2480 */ 2481 orig_oif = FIB_RES_OIF(*res); 2482 2483 fl4->flowi4_oif = dev_out->ifindex; 2484 flags |= RTCF_LOCAL; 2485 goto make_route; 2486 } 2487 2488 fib_select_path(net, res, fl4, skb); 2489 2490 dev_out = FIB_RES_DEV(*res); 2491 fl4->flowi4_oif = dev_out->ifindex; 2492 2493 2494 make_route: 2495 rth = __mkroute_output(res, fl4, orig_oif, dev_out, flags); 2496 2497 out: 2498 return rth; 2499 } 2500 2501 static struct dst_entry *ipv4_blackhole_dst_check(struct dst_entry *dst, u32 cookie) 2502 { 2503 return NULL; 2504 } 2505 2506 static unsigned int ipv4_blackhole_mtu(const struct dst_entry *dst) 2507 { 2508 unsigned int mtu = dst_metric_raw(dst, RTAX_MTU); 2509 2510 return mtu ? : dst->dev->mtu; 2511 } 2512 2513 static void ipv4_rt_blackhole_update_pmtu(struct dst_entry *dst, struct sock *sk, 2514 struct sk_buff *skb, u32 mtu) 2515 { 2516 } 2517 2518 static void ipv4_rt_blackhole_redirect(struct dst_entry *dst, struct sock *sk, 2519 struct sk_buff *skb) 2520 { 2521 } 2522 2523 static u32 *ipv4_rt_blackhole_cow_metrics(struct dst_entry *dst, 2524 unsigned long old) 2525 { 2526 return NULL; 2527 } 2528 2529 static struct dst_ops ipv4_dst_blackhole_ops = { 2530 .family = AF_INET, 2531 .check = ipv4_blackhole_dst_check, 2532 .mtu = ipv4_blackhole_mtu, 2533 .default_advmss = ipv4_default_advmss, 2534 .update_pmtu = ipv4_rt_blackhole_update_pmtu, 2535 .redirect = ipv4_rt_blackhole_redirect, 2536 .cow_metrics = ipv4_rt_blackhole_cow_metrics, 2537 .neigh_lookup = ipv4_neigh_lookup, 2538 }; 2539 2540 struct dst_entry *ipv4_blackhole_route(struct net *net, struct dst_entry *dst_orig) 2541 { 2542 struct rtable *ort = (struct rtable *) dst_orig; 2543 struct rtable *rt; 2544 2545 rt = dst_alloc(&ipv4_dst_blackhole_ops, NULL, 1, DST_OBSOLETE_DEAD, 0); 2546 if (rt) { 2547 struct dst_entry *new = &rt->dst; 2548 2549 new->__use = 1; 2550 new->input = dst_discard; 2551 new->output = dst_discard_out; 2552 2553 new->dev = net->loopback_dev; 2554 if (new->dev) 2555 dev_hold(new->dev); 2556 2557 rt->rt_is_input = ort->rt_is_input; 2558 rt->rt_iif = ort->rt_iif; 2559 rt->rt_pmtu = ort->rt_pmtu; 2560 rt->rt_mtu_locked = ort->rt_mtu_locked; 2561 2562 rt->rt_genid = rt_genid_ipv4(net); 2563 rt->rt_flags = ort->rt_flags; 2564 rt->rt_type = ort->rt_type; 2565 rt->rt_gateway = ort->rt_gateway; 2566 rt->rt_uses_gateway = ort->rt_uses_gateway; 2567 2568 INIT_LIST_HEAD(&rt->rt_uncached); 2569 } 2570 2571 dst_release(dst_orig); 2572 2573 return rt ? &rt->dst : ERR_PTR(-ENOMEM); 2574 } 2575 2576 struct rtable *ip_route_output_flow(struct net *net, struct flowi4 *flp4, 2577 const struct sock *sk) 2578 { 2579 struct rtable *rt = __ip_route_output_key(net, flp4); 2580 2581 if (IS_ERR(rt)) 2582 return rt; 2583 2584 if (flp4->flowi4_proto) 2585 rt = (struct rtable *)xfrm_lookup_route(net, &rt->dst, 2586 flowi4_to_flowi(flp4), 2587 sk, 0); 2588 2589 return rt; 2590 } 2591 EXPORT_SYMBOL_GPL(ip_route_output_flow); 2592 2593 /* called with rcu_read_lock held */ 2594 static int rt_fill_info(struct net *net, __be32 dst, __be32 src, 2595 struct rtable *rt, u32 table_id, struct flowi4 *fl4, 2596 struct sk_buff *skb, u32 portid, u32 seq) 2597 { 2598 struct rtmsg *r; 2599 struct nlmsghdr *nlh; 2600 unsigned long expires = 0; 2601 u32 error; 2602 u32 metrics[RTAX_MAX]; 2603 2604 nlh = nlmsg_put(skb, portid, seq, RTM_NEWROUTE, sizeof(*r), 0); 2605 if (!nlh) 2606 return -EMSGSIZE; 2607 2608 r = nlmsg_data(nlh); 2609 r->rtm_family = AF_INET; 2610 r->rtm_dst_len = 32; 2611 r->rtm_src_len = 0; 2612 r->rtm_tos = fl4->flowi4_tos; 2613 r->rtm_table = table_id < 256 ? table_id : RT_TABLE_COMPAT; 2614 if (nla_put_u32(skb, RTA_TABLE, table_id)) 2615 goto nla_put_failure; 2616 r->rtm_type = rt->rt_type; 2617 r->rtm_scope = RT_SCOPE_UNIVERSE; 2618 r->rtm_protocol = RTPROT_UNSPEC; 2619 r->rtm_flags = (rt->rt_flags & ~0xFFFF) | RTM_F_CLONED; 2620 if (rt->rt_flags & RTCF_NOTIFY) 2621 r->rtm_flags |= RTM_F_NOTIFY; 2622 if (IPCB(skb)->flags & IPSKB_DOREDIRECT) 2623 r->rtm_flags |= RTCF_DOREDIRECT; 2624 2625 if (nla_put_in_addr(skb, RTA_DST, dst)) 2626 goto nla_put_failure; 2627 if (src) { 2628 r->rtm_src_len = 32; 2629 if (nla_put_in_addr(skb, RTA_SRC, src)) 2630 goto nla_put_failure; 2631 } 2632 if (rt->dst.dev && 2633 nla_put_u32(skb, RTA_OIF, rt->dst.dev->ifindex)) 2634 goto nla_put_failure; 2635 #ifdef CONFIG_IP_ROUTE_CLASSID 2636 if (rt->dst.tclassid && 2637 nla_put_u32(skb, RTA_FLOW, rt->dst.tclassid)) 2638 goto nla_put_failure; 2639 #endif 2640 if (!rt_is_input_route(rt) && 2641 fl4->saddr != src) { 2642 if (nla_put_in_addr(skb, RTA_PREFSRC, fl4->saddr)) 2643 goto nla_put_failure; 2644 } 2645 if (rt->rt_uses_gateway && 2646 nla_put_in_addr(skb, RTA_GATEWAY, rt->rt_gateway)) 2647 goto nla_put_failure; 2648 2649 expires = rt->dst.expires; 2650 if (expires) { 2651 unsigned long now = jiffies; 2652 2653 if (time_before(now, expires)) 2654 expires -= now; 2655 else 2656 expires = 0; 2657 } 2658 2659 memcpy(metrics, dst_metrics_ptr(&rt->dst), sizeof(metrics)); 2660 if (rt->rt_pmtu && expires) 2661 metrics[RTAX_MTU - 1] = rt->rt_pmtu; 2662 if (rt->rt_mtu_locked && expires) 2663 metrics[RTAX_LOCK - 1] |= BIT(RTAX_MTU); 2664 if (rtnetlink_put_metrics(skb, metrics) < 0) 2665 goto nla_put_failure; 2666 2667 if (fl4->flowi4_mark && 2668 nla_put_u32(skb, RTA_MARK, fl4->flowi4_mark)) 2669 goto nla_put_failure; 2670 2671 if (!uid_eq(fl4->flowi4_uid, INVALID_UID) && 2672 nla_put_u32(skb, RTA_UID, 2673 from_kuid_munged(current_user_ns(), fl4->flowi4_uid))) 2674 goto nla_put_failure; 2675 2676 error = rt->dst.error; 2677 2678 if (rt_is_input_route(rt)) { 2679 #ifdef CONFIG_IP_MROUTE 2680 if (ipv4_is_multicast(dst) && !ipv4_is_local_multicast(dst) && 2681 IPV4_DEVCONF_ALL(net, MC_FORWARDING)) { 2682 int err = ipmr_get_route(net, skb, 2683 fl4->saddr, fl4->daddr, 2684 r, portid); 2685 2686 if (err <= 0) { 2687 if (err == 0) 2688 return 0; 2689 goto nla_put_failure; 2690 } 2691 } else 2692 #endif 2693 if (nla_put_u32(skb, RTA_IIF, fl4->flowi4_iif)) 2694 goto nla_put_failure; 2695 } 2696 2697 if (rtnl_put_cacheinfo(skb, &rt->dst, 0, expires, error) < 0) 2698 goto nla_put_failure; 2699 2700 nlmsg_end(skb, nlh); 2701 return 0; 2702 2703 nla_put_failure: 2704 nlmsg_cancel(skb, nlh); 2705 return -EMSGSIZE; 2706 } 2707 2708 static struct sk_buff *inet_rtm_getroute_build_skb(__be32 src, __be32 dst, 2709 u8 ip_proto, __be16 sport, 2710 __be16 dport) 2711 { 2712 struct sk_buff *skb; 2713 struct iphdr *iph; 2714 2715 skb = alloc_skb(NLMSG_GOODSIZE, GFP_KERNEL); 2716 if (!skb) 2717 return NULL; 2718 2719 /* Reserve room for dummy headers, this skb can pass 2720 * through good chunk of routing engine. 2721 */ 2722 skb_reset_mac_header(skb); 2723 skb_reset_network_header(skb); 2724 skb->protocol = htons(ETH_P_IP); 2725 iph = skb_put(skb, sizeof(struct iphdr)); 2726 iph->protocol = ip_proto; 2727 iph->saddr = src; 2728 iph->daddr = dst; 2729 iph->version = 0x4; 2730 iph->frag_off = 0; 2731 iph->ihl = 0x5; 2732 skb_set_transport_header(skb, skb->len); 2733 2734 switch (iph->protocol) { 2735 case IPPROTO_UDP: { 2736 struct udphdr *udph; 2737 2738 udph = skb_put_zero(skb, sizeof(struct udphdr)); 2739 udph->source = sport; 2740 udph->dest = dport; 2741 udph->len = sizeof(struct udphdr); 2742 udph->check = 0; 2743 break; 2744 } 2745 case IPPROTO_TCP: { 2746 struct tcphdr *tcph; 2747 2748 tcph = skb_put_zero(skb, sizeof(struct tcphdr)); 2749 tcph->source = sport; 2750 tcph->dest = dport; 2751 tcph->doff = sizeof(struct tcphdr) / 4; 2752 tcph->rst = 1; 2753 tcph->check = ~tcp_v4_check(sizeof(struct tcphdr), 2754 src, dst, 0); 2755 break; 2756 } 2757 case IPPROTO_ICMP: { 2758 struct icmphdr *icmph; 2759 2760 icmph = skb_put_zero(skb, sizeof(struct icmphdr)); 2761 icmph->type = ICMP_ECHO; 2762 icmph->code = 0; 2763 } 2764 } 2765 2766 return skb; 2767 } 2768 2769 static int inet_rtm_getroute(struct sk_buff *in_skb, struct nlmsghdr *nlh, 2770 struct netlink_ext_ack *extack) 2771 { 2772 struct net *net = sock_net(in_skb->sk); 2773 struct nlattr *tb[RTA_MAX+1]; 2774 u32 table_id = RT_TABLE_MAIN; 2775 __be16 sport = 0, dport = 0; 2776 struct fib_result res = {}; 2777 u8 ip_proto = IPPROTO_UDP; 2778 struct rtable *rt = NULL; 2779 struct sk_buff *skb; 2780 struct rtmsg *rtm; 2781 struct flowi4 fl4 = {}; 2782 __be32 dst = 0; 2783 __be32 src = 0; 2784 kuid_t uid; 2785 u32 iif; 2786 int err; 2787 int mark; 2788 2789 err = nlmsg_parse(nlh, sizeof(*rtm), tb, RTA_MAX, rtm_ipv4_policy, 2790 extack); 2791 if (err < 0) 2792 return err; 2793 2794 rtm = nlmsg_data(nlh); 2795 src = tb[RTA_SRC] ? nla_get_in_addr(tb[RTA_SRC]) : 0; 2796 dst = tb[RTA_DST] ? nla_get_in_addr(tb[RTA_DST]) : 0; 2797 iif = tb[RTA_IIF] ? nla_get_u32(tb[RTA_IIF]) : 0; 2798 mark = tb[RTA_MARK] ? nla_get_u32(tb[RTA_MARK]) : 0; 2799 if (tb[RTA_UID]) 2800 uid = make_kuid(current_user_ns(), nla_get_u32(tb[RTA_UID])); 2801 else 2802 uid = (iif ? INVALID_UID : current_uid()); 2803 2804 if (tb[RTA_IP_PROTO]) { 2805 err = rtm_getroute_parse_ip_proto(tb[RTA_IP_PROTO], 2806 &ip_proto, extack); 2807 if (err) 2808 return err; 2809 } 2810 2811 if (tb[RTA_SPORT]) 2812 sport = nla_get_be16(tb[RTA_SPORT]); 2813 2814 if (tb[RTA_DPORT]) 2815 dport = nla_get_be16(tb[RTA_DPORT]); 2816 2817 skb = inet_rtm_getroute_build_skb(src, dst, ip_proto, sport, dport); 2818 if (!skb) 2819 return -ENOBUFS; 2820 2821 fl4.daddr = dst; 2822 fl4.saddr = src; 2823 fl4.flowi4_tos = rtm->rtm_tos; 2824 fl4.flowi4_oif = tb[RTA_OIF] ? nla_get_u32(tb[RTA_OIF]) : 0; 2825 fl4.flowi4_mark = mark; 2826 fl4.flowi4_uid = uid; 2827 if (sport) 2828 fl4.fl4_sport = sport; 2829 if (dport) 2830 fl4.fl4_dport = dport; 2831 fl4.flowi4_proto = ip_proto; 2832 2833 rcu_read_lock(); 2834 2835 if (iif) { 2836 struct net_device *dev; 2837 2838 dev = dev_get_by_index_rcu(net, iif); 2839 if (!dev) { 2840 err = -ENODEV; 2841 goto errout_rcu; 2842 } 2843 2844 fl4.flowi4_iif = iif; /* for rt_fill_info */ 2845 skb->dev = dev; 2846 skb->mark = mark; 2847 err = ip_route_input_rcu(skb, dst, src, rtm->rtm_tos, 2848 dev, &res); 2849 2850 rt = skb_rtable(skb); 2851 if (err == 0 && rt->dst.error) 2852 err = -rt->dst.error; 2853 } else { 2854 fl4.flowi4_iif = LOOPBACK_IFINDEX; 2855 skb->dev = net->loopback_dev; 2856 rt = ip_route_output_key_hash_rcu(net, &fl4, &res, skb); 2857 err = 0; 2858 if (IS_ERR(rt)) 2859 err = PTR_ERR(rt); 2860 else 2861 skb_dst_set(skb, &rt->dst); 2862 } 2863 2864 if (err) 2865 goto errout_rcu; 2866 2867 if (rtm->rtm_flags & RTM_F_NOTIFY) 2868 rt->rt_flags |= RTCF_NOTIFY; 2869 2870 if (rtm->rtm_flags & RTM_F_LOOKUP_TABLE) 2871 table_id = res.table ? res.table->tb_id : 0; 2872 2873 /* reset skb for netlink reply msg */ 2874 skb_trim(skb, 0); 2875 skb_reset_network_header(skb); 2876 skb_reset_transport_header(skb); 2877 skb_reset_mac_header(skb); 2878 2879 if (rtm->rtm_flags & RTM_F_FIB_MATCH) { 2880 if (!res.fi) { 2881 err = fib_props[res.type].error; 2882 if (!err) 2883 err = -EHOSTUNREACH; 2884 goto errout_rcu; 2885 } 2886 err = fib_dump_info(skb, NETLINK_CB(in_skb).portid, 2887 nlh->nlmsg_seq, RTM_NEWROUTE, table_id, 2888 rt->rt_type, res.prefix, res.prefixlen, 2889 fl4.flowi4_tos, res.fi, 0); 2890 } else { 2891 err = rt_fill_info(net, dst, src, rt, table_id, &fl4, skb, 2892 NETLINK_CB(in_skb).portid, nlh->nlmsg_seq); 2893 } 2894 if (err < 0) 2895 goto errout_rcu; 2896 2897 rcu_read_unlock(); 2898 2899 err = rtnl_unicast(skb, net, NETLINK_CB(in_skb).portid); 2900 2901 errout_free: 2902 return err; 2903 errout_rcu: 2904 rcu_read_unlock(); 2905 kfree_skb(skb); 2906 goto errout_free; 2907 } 2908 2909 void ip_rt_multicast_event(struct in_device *in_dev) 2910 { 2911 rt_cache_flush(dev_net(in_dev->dev)); 2912 } 2913 2914 #ifdef CONFIG_SYSCTL 2915 static int ip_rt_gc_interval __read_mostly = 60 * HZ; 2916 static int ip_rt_gc_min_interval __read_mostly = HZ / 2; 2917 static int ip_rt_gc_elasticity __read_mostly = 8; 2918 static int ip_min_valid_pmtu __read_mostly = IPV4_MIN_MTU; 2919 2920 static int ipv4_sysctl_rtcache_flush(struct ctl_table *__ctl, int write, 2921 void __user *buffer, 2922 size_t *lenp, loff_t *ppos) 2923 { 2924 struct net *net = (struct net *)__ctl->extra1; 2925 2926 if (write) { 2927 rt_cache_flush(net); 2928 fnhe_genid_bump(net); 2929 return 0; 2930 } 2931 2932 return -EINVAL; 2933 } 2934 2935 static struct ctl_table ipv4_route_table[] = { 2936 { 2937 .procname = "gc_thresh", 2938 .data = &ipv4_dst_ops.gc_thresh, 2939 .maxlen = sizeof(int), 2940 .mode = 0644, 2941 .proc_handler = proc_dointvec, 2942 }, 2943 { 2944 .procname = "max_size", 2945 .data = &ip_rt_max_size, 2946 .maxlen = sizeof(int), 2947 .mode = 0644, 2948 .proc_handler = proc_dointvec, 2949 }, 2950 { 2951 /* Deprecated. Use gc_min_interval_ms */ 2952 2953 .procname = "gc_min_interval", 2954 .data = &ip_rt_gc_min_interval, 2955 .maxlen = sizeof(int), 2956 .mode = 0644, 2957 .proc_handler = proc_dointvec_jiffies, 2958 }, 2959 { 2960 .procname = "gc_min_interval_ms", 2961 .data = &ip_rt_gc_min_interval, 2962 .maxlen = sizeof(int), 2963 .mode = 0644, 2964 .proc_handler = proc_dointvec_ms_jiffies, 2965 }, 2966 { 2967 .procname = "gc_timeout", 2968 .data = &ip_rt_gc_timeout, 2969 .maxlen = sizeof(int), 2970 .mode = 0644, 2971 .proc_handler = proc_dointvec_jiffies, 2972 }, 2973 { 2974 .procname = "gc_interval", 2975 .data = &ip_rt_gc_interval, 2976 .maxlen = sizeof(int), 2977 .mode = 0644, 2978 .proc_handler = proc_dointvec_jiffies, 2979 }, 2980 { 2981 .procname = "redirect_load", 2982 .data = &ip_rt_redirect_load, 2983 .maxlen = sizeof(int), 2984 .mode = 0644, 2985 .proc_handler = proc_dointvec, 2986 }, 2987 { 2988 .procname = "redirect_number", 2989 .data = &ip_rt_redirect_number, 2990 .maxlen = sizeof(int), 2991 .mode = 0644, 2992 .proc_handler = proc_dointvec, 2993 }, 2994 { 2995 .procname = "redirect_silence", 2996 .data = &ip_rt_redirect_silence, 2997 .maxlen = sizeof(int), 2998 .mode = 0644, 2999 .proc_handler = proc_dointvec, 3000 }, 3001 { 3002 .procname = "error_cost", 3003 .data = &ip_rt_error_cost, 3004 .maxlen = sizeof(int), 3005 .mode = 0644, 3006 .proc_handler = proc_dointvec, 3007 }, 3008 { 3009 .procname = "error_burst", 3010 .data = &ip_rt_error_burst, 3011 .maxlen = sizeof(int), 3012 .mode = 0644, 3013 .proc_handler = proc_dointvec, 3014 }, 3015 { 3016 .procname = "gc_elasticity", 3017 .data = &ip_rt_gc_elasticity, 3018 .maxlen = sizeof(int), 3019 .mode = 0644, 3020 .proc_handler = proc_dointvec, 3021 }, 3022 { 3023 .procname = "mtu_expires", 3024 .data = &ip_rt_mtu_expires, 3025 .maxlen = sizeof(int), 3026 .mode = 0644, 3027 .proc_handler = proc_dointvec_jiffies, 3028 }, 3029 { 3030 .procname = "min_pmtu", 3031 .data = &ip_rt_min_pmtu, 3032 .maxlen = sizeof(int), 3033 .mode = 0644, 3034 .proc_handler = proc_dointvec_minmax, 3035 .extra1 = &ip_min_valid_pmtu, 3036 }, 3037 { 3038 .procname = "min_adv_mss", 3039 .data = &ip_rt_min_advmss, 3040 .maxlen = sizeof(int), 3041 .mode = 0644, 3042 .proc_handler = proc_dointvec, 3043 }, 3044 { } 3045 }; 3046 3047 static struct ctl_table ipv4_route_flush_table[] = { 3048 { 3049 .procname = "flush", 3050 .maxlen = sizeof(int), 3051 .mode = 0200, 3052 .proc_handler = ipv4_sysctl_rtcache_flush, 3053 }, 3054 { }, 3055 }; 3056 3057 static __net_init int sysctl_route_net_init(struct net *net) 3058 { 3059 struct ctl_table *tbl; 3060 3061 tbl = ipv4_route_flush_table; 3062 if (!net_eq(net, &init_net)) { 3063 tbl = kmemdup(tbl, sizeof(ipv4_route_flush_table), GFP_KERNEL); 3064 if (!tbl) 3065 goto err_dup; 3066 3067 /* Don't export sysctls to unprivileged users */ 3068 if (net->user_ns != &init_user_ns) 3069 tbl[0].procname = NULL; 3070 } 3071 tbl[0].extra1 = net; 3072 3073 net->ipv4.route_hdr = register_net_sysctl(net, "net/ipv4/route", tbl); 3074 if (!net->ipv4.route_hdr) 3075 goto err_reg; 3076 return 0; 3077 3078 err_reg: 3079 if (tbl != ipv4_route_flush_table) 3080 kfree(tbl); 3081 err_dup: 3082 return -ENOMEM; 3083 } 3084 3085 static __net_exit void sysctl_route_net_exit(struct net *net) 3086 { 3087 struct ctl_table *tbl; 3088 3089 tbl = net->ipv4.route_hdr->ctl_table_arg; 3090 unregister_net_sysctl_table(net->ipv4.route_hdr); 3091 BUG_ON(tbl == ipv4_route_flush_table); 3092 kfree(tbl); 3093 } 3094 3095 static __net_initdata struct pernet_operations sysctl_route_ops = { 3096 .init = sysctl_route_net_init, 3097 .exit = sysctl_route_net_exit, 3098 }; 3099 #endif 3100 3101 static __net_init int rt_genid_init(struct net *net) 3102 { 3103 atomic_set(&net->ipv4.rt_genid, 0); 3104 atomic_set(&net->fnhe_genid, 0); 3105 atomic_set(&net->ipv4.dev_addr_genid, get_random_int()); 3106 return 0; 3107 } 3108 3109 static __net_initdata struct pernet_operations rt_genid_ops = { 3110 .init = rt_genid_init, 3111 }; 3112 3113 static int __net_init ipv4_inetpeer_init(struct net *net) 3114 { 3115 struct inet_peer_base *bp = kmalloc(sizeof(*bp), GFP_KERNEL); 3116 3117 if (!bp) 3118 return -ENOMEM; 3119 inet_peer_base_init(bp); 3120 net->ipv4.peers = bp; 3121 return 0; 3122 } 3123 3124 static void __net_exit ipv4_inetpeer_exit(struct net *net) 3125 { 3126 struct inet_peer_base *bp = net->ipv4.peers; 3127 3128 net->ipv4.peers = NULL; 3129 inetpeer_invalidate_tree(bp); 3130 kfree(bp); 3131 } 3132 3133 static __net_initdata struct pernet_operations ipv4_inetpeer_ops = { 3134 .init = ipv4_inetpeer_init, 3135 .exit = ipv4_inetpeer_exit, 3136 }; 3137 3138 #ifdef CONFIG_IP_ROUTE_CLASSID 3139 struct ip_rt_acct __percpu *ip_rt_acct __read_mostly; 3140 #endif /* CONFIG_IP_ROUTE_CLASSID */ 3141 3142 int __init ip_rt_init(void) 3143 { 3144 int cpu; 3145 3146 ip_idents = kmalloc_array(IP_IDENTS_SZ, sizeof(*ip_idents), 3147 GFP_KERNEL); 3148 if (!ip_idents) 3149 panic("IP: failed to allocate ip_idents\n"); 3150 3151 prandom_bytes(ip_idents, IP_IDENTS_SZ * sizeof(*ip_idents)); 3152 3153 ip_tstamps = kcalloc(IP_IDENTS_SZ, sizeof(*ip_tstamps), GFP_KERNEL); 3154 if (!ip_tstamps) 3155 panic("IP: failed to allocate ip_tstamps\n"); 3156 3157 for_each_possible_cpu(cpu) { 3158 struct uncached_list *ul = &per_cpu(rt_uncached_list, cpu); 3159 3160 INIT_LIST_HEAD(&ul->head); 3161 spin_lock_init(&ul->lock); 3162 } 3163 #ifdef CONFIG_IP_ROUTE_CLASSID 3164 ip_rt_acct = __alloc_percpu(256 * sizeof(struct ip_rt_acct), __alignof__(struct ip_rt_acct)); 3165 if (!ip_rt_acct) 3166 panic("IP: failed to allocate ip_rt_acct\n"); 3167 #endif 3168 3169 ipv4_dst_ops.kmem_cachep = 3170 kmem_cache_create("ip_dst_cache", sizeof(struct rtable), 0, 3171 SLAB_HWCACHE_ALIGN|SLAB_PANIC, NULL); 3172 3173 ipv4_dst_blackhole_ops.kmem_cachep = ipv4_dst_ops.kmem_cachep; 3174 3175 if (dst_entries_init(&ipv4_dst_ops) < 0) 3176 panic("IP: failed to allocate ipv4_dst_ops counter\n"); 3177 3178 if (dst_entries_init(&ipv4_dst_blackhole_ops) < 0) 3179 panic("IP: failed to allocate ipv4_dst_blackhole_ops counter\n"); 3180 3181 ipv4_dst_ops.gc_thresh = ~0; 3182 ip_rt_max_size = INT_MAX; 3183 3184 devinet_init(); 3185 ip_fib_init(); 3186 3187 if (ip_rt_proc_init()) 3188 pr_err("Unable to create route proc files\n"); 3189 #ifdef CONFIG_XFRM 3190 xfrm_init(); 3191 xfrm4_init(); 3192 #endif 3193 rtnl_register(PF_INET, RTM_GETROUTE, inet_rtm_getroute, NULL, 3194 RTNL_FLAG_DOIT_UNLOCKED); 3195 3196 #ifdef CONFIG_SYSCTL 3197 register_pernet_subsys(&sysctl_route_ops); 3198 #endif 3199 register_pernet_subsys(&rt_genid_ops); 3200 register_pernet_subsys(&ipv4_inetpeer_ops); 3201 return 0; 3202 } 3203 3204 #ifdef CONFIG_SYSCTL 3205 /* 3206 * We really need to sanitize the damn ipv4 init order, then all 3207 * this nonsense will go away. 3208 */ 3209 void __init ip_static_sysctl_init(void) 3210 { 3211 register_net_sysctl(&init_net, "net/ipv4/route", ipv4_route_table); 3212 } 3213 #endif 3214