1 /* 2 * INET An implementation of the TCP/IP protocol suite for the LINUX 3 * operating system. INET is implemented using the BSD Socket 4 * interface as the means of communication with the user level. 5 * 6 * ROUTE - implementation of the IP router. 7 * 8 * Authors: Ross Biro 9 * Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG> 10 * Alan Cox, <gw4pts@gw4pts.ampr.org> 11 * Linus Torvalds, <Linus.Torvalds@helsinki.fi> 12 * Alexey Kuznetsov, <kuznet@ms2.inr.ac.ru> 13 * 14 * Fixes: 15 * Alan Cox : Verify area fixes. 16 * Alan Cox : cli() protects routing changes 17 * Rui Oliveira : ICMP routing table updates 18 * (rco@di.uminho.pt) Routing table insertion and update 19 * Linus Torvalds : Rewrote bits to be sensible 20 * Alan Cox : Added BSD route gw semantics 21 * Alan Cox : Super /proc >4K 22 * Alan Cox : MTU in route table 23 * Alan Cox : MSS actually. Also added the window 24 * clamper. 25 * Sam Lantinga : Fixed route matching in rt_del() 26 * Alan Cox : Routing cache support. 27 * Alan Cox : Removed compatibility cruft. 28 * Alan Cox : RTF_REJECT support. 29 * Alan Cox : TCP irtt support. 30 * Jonathan Naylor : Added Metric support. 31 * Miquel van Smoorenburg : BSD API fixes. 32 * Miquel van Smoorenburg : Metrics. 33 * Alan Cox : Use __u32 properly 34 * Alan Cox : Aligned routing errors more closely with BSD 35 * our system is still very different. 36 * Alan Cox : Faster /proc handling 37 * Alexey Kuznetsov : Massive rework to support tree based routing, 38 * routing caches and better behaviour. 39 * 40 * Olaf Erb : irtt wasn't being copied right. 41 * Bjorn Ekwall : Kerneld route support. 42 * Alan Cox : Multicast fixed (I hope) 43 * Pavel Krauz : Limited broadcast fixed 44 * Mike McLagan : Routing by source 45 * Alexey Kuznetsov : End of old history. Split to fib.c and 46 * route.c and rewritten from scratch. 47 * Andi Kleen : Load-limit warning messages. 48 * Vitaly E. Lavrov : Transparent proxy revived after year coma. 49 * Vitaly E. Lavrov : Race condition in ip_route_input_slow. 50 * Tobias Ringstrom : Uninitialized res.type in ip_route_output_slow. 51 * Vladimir V. Ivanov : IP rule info (flowid) is really useful. 52 * Marc Boucher : routing by fwmark 53 * Robert Olsson : Added rt_cache statistics 54 * Arnaldo C. Melo : Convert proc stuff to seq_file 55 * Eric Dumazet : hashed spinlocks and rt_check_expire() fixes. 56 * Ilia Sotnikov : Ignore TOS on PMTUD and Redirect 57 * Ilia Sotnikov : Removed TOS from hash calculations 58 * 59 * This program is free software; you can redistribute it and/or 60 * modify it under the terms of the GNU General Public License 61 * as published by the Free Software Foundation; either version 62 * 2 of the License, or (at your option) any later version. 63 */ 64 65 #define pr_fmt(fmt) "IPv4: " fmt 66 67 #include <linux/module.h> 68 #include <linux/uaccess.h> 69 #include <linux/bitops.h> 70 #include <linux/types.h> 71 #include <linux/kernel.h> 72 #include <linux/mm.h> 73 #include <linux/string.h> 74 #include <linux/socket.h> 75 #include <linux/sockios.h> 76 #include <linux/errno.h> 77 #include <linux/in.h> 78 #include <linux/inet.h> 79 #include <linux/netdevice.h> 80 #include <linux/proc_fs.h> 81 #include <linux/init.h> 82 #include <linux/skbuff.h> 83 #include <linux/inetdevice.h> 84 #include <linux/igmp.h> 85 #include <linux/pkt_sched.h> 86 #include <linux/mroute.h> 87 #include <linux/netfilter_ipv4.h> 88 #include <linux/random.h> 89 #include <linux/rcupdate.h> 90 #include <linux/times.h> 91 #include <linux/slab.h> 92 #include <linux/jhash.h> 93 #include <net/dst.h> 94 #include <net/dst_metadata.h> 95 #include <net/net_namespace.h> 96 #include <net/protocol.h> 97 #include <net/ip.h> 98 #include <net/route.h> 99 #include <net/inetpeer.h> 100 #include <net/sock.h> 101 #include <net/ip_fib.h> 102 #include <net/arp.h> 103 #include <net/tcp.h> 104 #include <net/icmp.h> 105 #include <net/xfrm.h> 106 #include <net/lwtunnel.h> 107 #include <net/netevent.h> 108 #include <net/rtnetlink.h> 109 #ifdef CONFIG_SYSCTL 110 #include <linux/sysctl.h> 111 #include <linux/kmemleak.h> 112 #endif 113 #include <net/secure_seq.h> 114 #include <net/ip_tunnels.h> 115 #include <net/l3mdev.h> 116 117 #include "fib_lookup.h" 118 119 #define RT_FL_TOS(oldflp4) \ 120 ((oldflp4)->flowi4_tos & (IPTOS_RT_MASK | RTO_ONLINK)) 121 122 #define RT_GC_TIMEOUT (300*HZ) 123 124 static int ip_rt_max_size; 125 static int ip_rt_redirect_number __read_mostly = 9; 126 static int ip_rt_redirect_load __read_mostly = HZ / 50; 127 static int ip_rt_redirect_silence __read_mostly = ((HZ / 50) << (9 + 1)); 128 static int ip_rt_error_cost __read_mostly = HZ; 129 static int ip_rt_error_burst __read_mostly = 5 * HZ; 130 static int ip_rt_mtu_expires __read_mostly = 10 * 60 * HZ; 131 static int ip_rt_min_pmtu __read_mostly = 512 + 20 + 20; 132 static int ip_rt_min_advmss __read_mostly = 256; 133 134 static int ip_rt_gc_timeout __read_mostly = RT_GC_TIMEOUT; 135 /* 136 * Interface to generic destination cache. 137 */ 138 139 static struct dst_entry *ipv4_dst_check(struct dst_entry *dst, u32 cookie); 140 static unsigned int ipv4_default_advmss(const struct dst_entry *dst); 141 static unsigned int ipv4_mtu(const struct dst_entry *dst); 142 static struct dst_entry *ipv4_negative_advice(struct dst_entry *dst); 143 static void ipv4_link_failure(struct sk_buff *skb); 144 static void ip_rt_update_pmtu(struct dst_entry *dst, struct sock *sk, 145 struct sk_buff *skb, u32 mtu); 146 static void ip_do_redirect(struct dst_entry *dst, struct sock *sk, 147 struct sk_buff *skb); 148 static void ipv4_dst_destroy(struct dst_entry *dst); 149 150 static u32 *ipv4_cow_metrics(struct dst_entry *dst, unsigned long old) 151 { 152 WARN_ON(1); 153 return NULL; 154 } 155 156 static struct neighbour *ipv4_neigh_lookup(const struct dst_entry *dst, 157 struct sk_buff *skb, 158 const void *daddr); 159 static void ipv4_confirm_neigh(const struct dst_entry *dst, const void *daddr); 160 161 static struct dst_ops ipv4_dst_ops = { 162 .family = AF_INET, 163 .check = ipv4_dst_check, 164 .default_advmss = ipv4_default_advmss, 165 .mtu = ipv4_mtu, 166 .cow_metrics = ipv4_cow_metrics, 167 .destroy = ipv4_dst_destroy, 168 .negative_advice = ipv4_negative_advice, 169 .link_failure = ipv4_link_failure, 170 .update_pmtu = ip_rt_update_pmtu, 171 .redirect = ip_do_redirect, 172 .local_out = __ip_local_out, 173 .neigh_lookup = ipv4_neigh_lookup, 174 .confirm_neigh = ipv4_confirm_neigh, 175 }; 176 177 #define ECN_OR_COST(class) TC_PRIO_##class 178 179 const __u8 ip_tos2prio[16] = { 180 TC_PRIO_BESTEFFORT, 181 ECN_OR_COST(BESTEFFORT), 182 TC_PRIO_BESTEFFORT, 183 ECN_OR_COST(BESTEFFORT), 184 TC_PRIO_BULK, 185 ECN_OR_COST(BULK), 186 TC_PRIO_BULK, 187 ECN_OR_COST(BULK), 188 TC_PRIO_INTERACTIVE, 189 ECN_OR_COST(INTERACTIVE), 190 TC_PRIO_INTERACTIVE, 191 ECN_OR_COST(INTERACTIVE), 192 TC_PRIO_INTERACTIVE_BULK, 193 ECN_OR_COST(INTERACTIVE_BULK), 194 TC_PRIO_INTERACTIVE_BULK, 195 ECN_OR_COST(INTERACTIVE_BULK) 196 }; 197 EXPORT_SYMBOL(ip_tos2prio); 198 199 static DEFINE_PER_CPU(struct rt_cache_stat, rt_cache_stat); 200 #define RT_CACHE_STAT_INC(field) raw_cpu_inc(rt_cache_stat.field) 201 202 #ifdef CONFIG_PROC_FS 203 static void *rt_cache_seq_start(struct seq_file *seq, loff_t *pos) 204 { 205 if (*pos) 206 return NULL; 207 return SEQ_START_TOKEN; 208 } 209 210 static void *rt_cache_seq_next(struct seq_file *seq, void *v, loff_t *pos) 211 { 212 ++*pos; 213 return NULL; 214 } 215 216 static void rt_cache_seq_stop(struct seq_file *seq, void *v) 217 { 218 } 219 220 static int rt_cache_seq_show(struct seq_file *seq, void *v) 221 { 222 if (v == SEQ_START_TOKEN) 223 seq_printf(seq, "%-127s\n", 224 "Iface\tDestination\tGateway \tFlags\t\tRefCnt\tUse\t" 225 "Metric\tSource\t\tMTU\tWindow\tIRTT\tTOS\tHHRef\t" 226 "HHUptod\tSpecDst"); 227 return 0; 228 } 229 230 static const struct seq_operations rt_cache_seq_ops = { 231 .start = rt_cache_seq_start, 232 .next = rt_cache_seq_next, 233 .stop = rt_cache_seq_stop, 234 .show = rt_cache_seq_show, 235 }; 236 237 static int rt_cache_seq_open(struct inode *inode, struct file *file) 238 { 239 return seq_open(file, &rt_cache_seq_ops); 240 } 241 242 static const struct file_operations rt_cache_seq_fops = { 243 .owner = THIS_MODULE, 244 .open = rt_cache_seq_open, 245 .read = seq_read, 246 .llseek = seq_lseek, 247 .release = seq_release, 248 }; 249 250 251 static void *rt_cpu_seq_start(struct seq_file *seq, loff_t *pos) 252 { 253 int cpu; 254 255 if (*pos == 0) 256 return SEQ_START_TOKEN; 257 258 for (cpu = *pos-1; cpu < nr_cpu_ids; ++cpu) { 259 if (!cpu_possible(cpu)) 260 continue; 261 *pos = cpu+1; 262 return &per_cpu(rt_cache_stat, cpu); 263 } 264 return NULL; 265 } 266 267 static void *rt_cpu_seq_next(struct seq_file *seq, void *v, loff_t *pos) 268 { 269 int cpu; 270 271 for (cpu = *pos; cpu < nr_cpu_ids; ++cpu) { 272 if (!cpu_possible(cpu)) 273 continue; 274 *pos = cpu+1; 275 return &per_cpu(rt_cache_stat, cpu); 276 } 277 return NULL; 278 279 } 280 281 static void rt_cpu_seq_stop(struct seq_file *seq, void *v) 282 { 283 284 } 285 286 static int rt_cpu_seq_show(struct seq_file *seq, void *v) 287 { 288 struct rt_cache_stat *st = v; 289 290 if (v == SEQ_START_TOKEN) { 291 seq_printf(seq, "entries in_hit in_slow_tot in_slow_mc in_no_route in_brd in_martian_dst in_martian_src out_hit out_slow_tot out_slow_mc gc_total gc_ignored gc_goal_miss gc_dst_overflow in_hlist_search out_hlist_search\n"); 292 return 0; 293 } 294 295 seq_printf(seq,"%08x %08x %08x %08x %08x %08x %08x %08x " 296 " %08x %08x %08x %08x %08x %08x %08x %08x %08x \n", 297 dst_entries_get_slow(&ipv4_dst_ops), 298 0, /* st->in_hit */ 299 st->in_slow_tot, 300 st->in_slow_mc, 301 st->in_no_route, 302 st->in_brd, 303 st->in_martian_dst, 304 st->in_martian_src, 305 306 0, /* st->out_hit */ 307 st->out_slow_tot, 308 st->out_slow_mc, 309 310 0, /* st->gc_total */ 311 0, /* st->gc_ignored */ 312 0, /* st->gc_goal_miss */ 313 0, /* st->gc_dst_overflow */ 314 0, /* st->in_hlist_search */ 315 0 /* st->out_hlist_search */ 316 ); 317 return 0; 318 } 319 320 static const struct seq_operations rt_cpu_seq_ops = { 321 .start = rt_cpu_seq_start, 322 .next = rt_cpu_seq_next, 323 .stop = rt_cpu_seq_stop, 324 .show = rt_cpu_seq_show, 325 }; 326 327 328 static int rt_cpu_seq_open(struct inode *inode, struct file *file) 329 { 330 return seq_open(file, &rt_cpu_seq_ops); 331 } 332 333 static const struct file_operations rt_cpu_seq_fops = { 334 .owner = THIS_MODULE, 335 .open = rt_cpu_seq_open, 336 .read = seq_read, 337 .llseek = seq_lseek, 338 .release = seq_release, 339 }; 340 341 #ifdef CONFIG_IP_ROUTE_CLASSID 342 static int rt_acct_proc_show(struct seq_file *m, void *v) 343 { 344 struct ip_rt_acct *dst, *src; 345 unsigned int i, j; 346 347 dst = kcalloc(256, sizeof(struct ip_rt_acct), GFP_KERNEL); 348 if (!dst) 349 return -ENOMEM; 350 351 for_each_possible_cpu(i) { 352 src = (struct ip_rt_acct *)per_cpu_ptr(ip_rt_acct, i); 353 for (j = 0; j < 256; j++) { 354 dst[j].o_bytes += src[j].o_bytes; 355 dst[j].o_packets += src[j].o_packets; 356 dst[j].i_bytes += src[j].i_bytes; 357 dst[j].i_packets += src[j].i_packets; 358 } 359 } 360 361 seq_write(m, dst, 256 * sizeof(struct ip_rt_acct)); 362 kfree(dst); 363 return 0; 364 } 365 366 static int rt_acct_proc_open(struct inode *inode, struct file *file) 367 { 368 return single_open(file, rt_acct_proc_show, NULL); 369 } 370 371 static const struct file_operations rt_acct_proc_fops = { 372 .owner = THIS_MODULE, 373 .open = rt_acct_proc_open, 374 .read = seq_read, 375 .llseek = seq_lseek, 376 .release = single_release, 377 }; 378 #endif 379 380 static int __net_init ip_rt_do_proc_init(struct net *net) 381 { 382 struct proc_dir_entry *pde; 383 384 pde = proc_create("rt_cache", S_IRUGO, net->proc_net, 385 &rt_cache_seq_fops); 386 if (!pde) 387 goto err1; 388 389 pde = proc_create("rt_cache", S_IRUGO, 390 net->proc_net_stat, &rt_cpu_seq_fops); 391 if (!pde) 392 goto err2; 393 394 #ifdef CONFIG_IP_ROUTE_CLASSID 395 pde = proc_create("rt_acct", 0, net->proc_net, &rt_acct_proc_fops); 396 if (!pde) 397 goto err3; 398 #endif 399 return 0; 400 401 #ifdef CONFIG_IP_ROUTE_CLASSID 402 err3: 403 remove_proc_entry("rt_cache", net->proc_net_stat); 404 #endif 405 err2: 406 remove_proc_entry("rt_cache", net->proc_net); 407 err1: 408 return -ENOMEM; 409 } 410 411 static void __net_exit ip_rt_do_proc_exit(struct net *net) 412 { 413 remove_proc_entry("rt_cache", net->proc_net_stat); 414 remove_proc_entry("rt_cache", net->proc_net); 415 #ifdef CONFIG_IP_ROUTE_CLASSID 416 remove_proc_entry("rt_acct", net->proc_net); 417 #endif 418 } 419 420 static struct pernet_operations ip_rt_proc_ops __net_initdata = { 421 .init = ip_rt_do_proc_init, 422 .exit = ip_rt_do_proc_exit, 423 }; 424 425 static int __init ip_rt_proc_init(void) 426 { 427 return register_pernet_subsys(&ip_rt_proc_ops); 428 } 429 430 #else 431 static inline int ip_rt_proc_init(void) 432 { 433 return 0; 434 } 435 #endif /* CONFIG_PROC_FS */ 436 437 static inline bool rt_is_expired(const struct rtable *rth) 438 { 439 return rth->rt_genid != rt_genid_ipv4(dev_net(rth->dst.dev)); 440 } 441 442 void rt_cache_flush(struct net *net) 443 { 444 rt_genid_bump_ipv4(net); 445 } 446 447 static struct neighbour *ipv4_neigh_lookup(const struct dst_entry *dst, 448 struct sk_buff *skb, 449 const void *daddr) 450 { 451 struct net_device *dev = dst->dev; 452 const __be32 *pkey = daddr; 453 const struct rtable *rt; 454 struct neighbour *n; 455 456 rt = (const struct rtable *) dst; 457 if (rt->rt_gateway) 458 pkey = (const __be32 *) &rt->rt_gateway; 459 else if (skb) 460 pkey = &ip_hdr(skb)->daddr; 461 462 n = __ipv4_neigh_lookup(dev, *(__force u32 *)pkey); 463 if (n) 464 return n; 465 return neigh_create(&arp_tbl, pkey, dev); 466 } 467 468 static void ipv4_confirm_neigh(const struct dst_entry *dst, const void *daddr) 469 { 470 struct net_device *dev = dst->dev; 471 const __be32 *pkey = daddr; 472 const struct rtable *rt; 473 474 rt = (const struct rtable *)dst; 475 if (rt->rt_gateway) 476 pkey = (const __be32 *)&rt->rt_gateway; 477 else if (!daddr || 478 (rt->rt_flags & 479 (RTCF_MULTICAST | RTCF_BROADCAST | RTCF_LOCAL))) 480 return; 481 482 __ipv4_confirm_neigh(dev, *(__force u32 *)pkey); 483 } 484 485 #define IP_IDENTS_SZ 2048u 486 487 static atomic_t *ip_idents __read_mostly; 488 static u32 *ip_tstamps __read_mostly; 489 490 /* In order to protect privacy, we add a perturbation to identifiers 491 * if one generator is seldom used. This makes hard for an attacker 492 * to infer how many packets were sent between two points in time. 493 */ 494 u32 ip_idents_reserve(u32 hash, int segs) 495 { 496 u32 *p_tstamp = ip_tstamps + hash % IP_IDENTS_SZ; 497 atomic_t *p_id = ip_idents + hash % IP_IDENTS_SZ; 498 u32 old = READ_ONCE(*p_tstamp); 499 u32 now = (u32)jiffies; 500 u32 new, delta = 0; 501 502 if (old != now && cmpxchg(p_tstamp, old, now) == old) 503 delta = prandom_u32_max(now - old); 504 505 /* Do not use atomic_add_return() as it makes UBSAN unhappy */ 506 do { 507 old = (u32)atomic_read(p_id); 508 new = old + delta + segs; 509 } while (atomic_cmpxchg(p_id, old, new) != old); 510 511 return new - segs; 512 } 513 EXPORT_SYMBOL(ip_idents_reserve); 514 515 void __ip_select_ident(struct net *net, struct iphdr *iph, int segs) 516 { 517 static u32 ip_idents_hashrnd __read_mostly; 518 u32 hash, id; 519 520 net_get_random_once(&ip_idents_hashrnd, sizeof(ip_idents_hashrnd)); 521 522 hash = jhash_3words((__force u32)iph->daddr, 523 (__force u32)iph->saddr, 524 iph->protocol ^ net_hash_mix(net), 525 ip_idents_hashrnd); 526 id = ip_idents_reserve(hash, segs); 527 iph->id = htons(id); 528 } 529 EXPORT_SYMBOL(__ip_select_ident); 530 531 static void __build_flow_key(const struct net *net, struct flowi4 *fl4, 532 const struct sock *sk, 533 const struct iphdr *iph, 534 int oif, u8 tos, 535 u8 prot, u32 mark, int flow_flags) 536 { 537 if (sk) { 538 const struct inet_sock *inet = inet_sk(sk); 539 540 oif = sk->sk_bound_dev_if; 541 mark = sk->sk_mark; 542 tos = RT_CONN_FLAGS(sk); 543 prot = inet->hdrincl ? IPPROTO_RAW : sk->sk_protocol; 544 } 545 flowi4_init_output(fl4, oif, mark, tos, 546 RT_SCOPE_UNIVERSE, prot, 547 flow_flags, 548 iph->daddr, iph->saddr, 0, 0, 549 sock_net_uid(net, sk)); 550 } 551 552 static void build_skb_flow_key(struct flowi4 *fl4, const struct sk_buff *skb, 553 const struct sock *sk) 554 { 555 const struct net *net = dev_net(skb->dev); 556 const struct iphdr *iph = ip_hdr(skb); 557 int oif = skb->dev->ifindex; 558 u8 tos = RT_TOS(iph->tos); 559 u8 prot = iph->protocol; 560 u32 mark = skb->mark; 561 562 __build_flow_key(net, fl4, sk, iph, oif, tos, prot, mark, 0); 563 } 564 565 static void build_sk_flow_key(struct flowi4 *fl4, const struct sock *sk) 566 { 567 const struct inet_sock *inet = inet_sk(sk); 568 const struct ip_options_rcu *inet_opt; 569 __be32 daddr = inet->inet_daddr; 570 571 rcu_read_lock(); 572 inet_opt = rcu_dereference(inet->inet_opt); 573 if (inet_opt && inet_opt->opt.srr) 574 daddr = inet_opt->opt.faddr; 575 flowi4_init_output(fl4, sk->sk_bound_dev_if, sk->sk_mark, 576 RT_CONN_FLAGS(sk), RT_SCOPE_UNIVERSE, 577 inet->hdrincl ? IPPROTO_RAW : sk->sk_protocol, 578 inet_sk_flowi_flags(sk), 579 daddr, inet->inet_saddr, 0, 0, sk->sk_uid); 580 rcu_read_unlock(); 581 } 582 583 static void ip_rt_build_flow_key(struct flowi4 *fl4, const struct sock *sk, 584 const struct sk_buff *skb) 585 { 586 if (skb) 587 build_skb_flow_key(fl4, skb, sk); 588 else 589 build_sk_flow_key(fl4, sk); 590 } 591 592 static DEFINE_SPINLOCK(fnhe_lock); 593 594 static void fnhe_flush_routes(struct fib_nh_exception *fnhe) 595 { 596 struct rtable *rt; 597 598 rt = rcu_dereference(fnhe->fnhe_rth_input); 599 if (rt) { 600 RCU_INIT_POINTER(fnhe->fnhe_rth_input, NULL); 601 dst_dev_put(&rt->dst); 602 dst_release(&rt->dst); 603 } 604 rt = rcu_dereference(fnhe->fnhe_rth_output); 605 if (rt) { 606 RCU_INIT_POINTER(fnhe->fnhe_rth_output, NULL); 607 dst_dev_put(&rt->dst); 608 dst_release(&rt->dst); 609 } 610 } 611 612 static struct fib_nh_exception *fnhe_oldest(struct fnhe_hash_bucket *hash) 613 { 614 struct fib_nh_exception *fnhe, *oldest; 615 616 oldest = rcu_dereference(hash->chain); 617 for (fnhe = rcu_dereference(oldest->fnhe_next); fnhe; 618 fnhe = rcu_dereference(fnhe->fnhe_next)) { 619 if (time_before(fnhe->fnhe_stamp, oldest->fnhe_stamp)) 620 oldest = fnhe; 621 } 622 fnhe_flush_routes(oldest); 623 return oldest; 624 } 625 626 static inline u32 fnhe_hashfun(__be32 daddr) 627 { 628 static u32 fnhe_hashrnd __read_mostly; 629 u32 hval; 630 631 net_get_random_once(&fnhe_hashrnd, sizeof(fnhe_hashrnd)); 632 hval = jhash_1word((__force u32) daddr, fnhe_hashrnd); 633 return hash_32(hval, FNHE_HASH_SHIFT); 634 } 635 636 static void fill_route_from_fnhe(struct rtable *rt, struct fib_nh_exception *fnhe) 637 { 638 rt->rt_pmtu = fnhe->fnhe_pmtu; 639 rt->dst.expires = fnhe->fnhe_expires; 640 641 if (fnhe->fnhe_gw) { 642 rt->rt_flags |= RTCF_REDIRECTED; 643 rt->rt_gateway = fnhe->fnhe_gw; 644 rt->rt_uses_gateway = 1; 645 } 646 } 647 648 static void update_or_create_fnhe(struct fib_nh *nh, __be32 daddr, __be32 gw, 649 u32 pmtu, unsigned long expires) 650 { 651 struct fnhe_hash_bucket *hash; 652 struct fib_nh_exception *fnhe; 653 struct rtable *rt; 654 u32 genid, hval; 655 unsigned int i; 656 int depth; 657 658 genid = fnhe_genid(dev_net(nh->nh_dev)); 659 hval = fnhe_hashfun(daddr); 660 661 spin_lock_bh(&fnhe_lock); 662 663 hash = rcu_dereference(nh->nh_exceptions); 664 if (!hash) { 665 hash = kzalloc(FNHE_HASH_SIZE * sizeof(*hash), GFP_ATOMIC); 666 if (!hash) 667 goto out_unlock; 668 rcu_assign_pointer(nh->nh_exceptions, hash); 669 } 670 671 hash += hval; 672 673 depth = 0; 674 for (fnhe = rcu_dereference(hash->chain); fnhe; 675 fnhe = rcu_dereference(fnhe->fnhe_next)) { 676 if (fnhe->fnhe_daddr == daddr) 677 break; 678 depth++; 679 } 680 681 if (fnhe) { 682 if (fnhe->fnhe_genid != genid) 683 fnhe->fnhe_genid = genid; 684 if (gw) 685 fnhe->fnhe_gw = gw; 686 if (pmtu) 687 fnhe->fnhe_pmtu = pmtu; 688 fnhe->fnhe_expires = max(1UL, expires); 689 /* Update all cached dsts too */ 690 rt = rcu_dereference(fnhe->fnhe_rth_input); 691 if (rt) 692 fill_route_from_fnhe(rt, fnhe); 693 rt = rcu_dereference(fnhe->fnhe_rth_output); 694 if (rt) 695 fill_route_from_fnhe(rt, fnhe); 696 } else { 697 if (depth > FNHE_RECLAIM_DEPTH) 698 fnhe = fnhe_oldest(hash); 699 else { 700 fnhe = kzalloc(sizeof(*fnhe), GFP_ATOMIC); 701 if (!fnhe) 702 goto out_unlock; 703 704 fnhe->fnhe_next = hash->chain; 705 rcu_assign_pointer(hash->chain, fnhe); 706 } 707 fnhe->fnhe_genid = genid; 708 fnhe->fnhe_daddr = daddr; 709 fnhe->fnhe_gw = gw; 710 fnhe->fnhe_pmtu = pmtu; 711 fnhe->fnhe_expires = expires; 712 713 /* Exception created; mark the cached routes for the nexthop 714 * stale, so anyone caching it rechecks if this exception 715 * applies to them. 716 */ 717 rt = rcu_dereference(nh->nh_rth_input); 718 if (rt) 719 rt->dst.obsolete = DST_OBSOLETE_KILL; 720 721 for_each_possible_cpu(i) { 722 struct rtable __rcu **prt; 723 prt = per_cpu_ptr(nh->nh_pcpu_rth_output, i); 724 rt = rcu_dereference(*prt); 725 if (rt) 726 rt->dst.obsolete = DST_OBSOLETE_KILL; 727 } 728 } 729 730 fnhe->fnhe_stamp = jiffies; 731 732 out_unlock: 733 spin_unlock_bh(&fnhe_lock); 734 } 735 736 static void __ip_do_redirect(struct rtable *rt, struct sk_buff *skb, struct flowi4 *fl4, 737 bool kill_route) 738 { 739 __be32 new_gw = icmp_hdr(skb)->un.gateway; 740 __be32 old_gw = ip_hdr(skb)->saddr; 741 struct net_device *dev = skb->dev; 742 struct in_device *in_dev; 743 struct fib_result res; 744 struct neighbour *n; 745 struct net *net; 746 747 switch (icmp_hdr(skb)->code & 7) { 748 case ICMP_REDIR_NET: 749 case ICMP_REDIR_NETTOS: 750 case ICMP_REDIR_HOST: 751 case ICMP_REDIR_HOSTTOS: 752 break; 753 754 default: 755 return; 756 } 757 758 if (rt->rt_gateway != old_gw) 759 return; 760 761 in_dev = __in_dev_get_rcu(dev); 762 if (!in_dev) 763 return; 764 765 net = dev_net(dev); 766 if (new_gw == old_gw || !IN_DEV_RX_REDIRECTS(in_dev) || 767 ipv4_is_multicast(new_gw) || ipv4_is_lbcast(new_gw) || 768 ipv4_is_zeronet(new_gw)) 769 goto reject_redirect; 770 771 if (!IN_DEV_SHARED_MEDIA(in_dev)) { 772 if (!inet_addr_onlink(in_dev, new_gw, old_gw)) 773 goto reject_redirect; 774 if (IN_DEV_SEC_REDIRECTS(in_dev) && ip_fib_check_default(new_gw, dev)) 775 goto reject_redirect; 776 } else { 777 if (inet_addr_type(net, new_gw) != RTN_UNICAST) 778 goto reject_redirect; 779 } 780 781 n = __ipv4_neigh_lookup(rt->dst.dev, new_gw); 782 if (!n) 783 n = neigh_create(&arp_tbl, &new_gw, rt->dst.dev); 784 if (!IS_ERR(n)) { 785 if (!(n->nud_state & NUD_VALID)) { 786 neigh_event_send(n, NULL); 787 } else { 788 if (fib_lookup(net, fl4, &res, 0) == 0) { 789 struct fib_nh *nh = &FIB_RES_NH(res); 790 791 update_or_create_fnhe(nh, fl4->daddr, new_gw, 792 0, jiffies + ip_rt_gc_timeout); 793 } 794 if (kill_route) 795 rt->dst.obsolete = DST_OBSOLETE_KILL; 796 call_netevent_notifiers(NETEVENT_NEIGH_UPDATE, n); 797 } 798 neigh_release(n); 799 } 800 return; 801 802 reject_redirect: 803 #ifdef CONFIG_IP_ROUTE_VERBOSE 804 if (IN_DEV_LOG_MARTIANS(in_dev)) { 805 const struct iphdr *iph = (const struct iphdr *) skb->data; 806 __be32 daddr = iph->daddr; 807 __be32 saddr = iph->saddr; 808 809 net_info_ratelimited("Redirect from %pI4 on %s about %pI4 ignored\n" 810 " Advised path = %pI4 -> %pI4\n", 811 &old_gw, dev->name, &new_gw, 812 &saddr, &daddr); 813 } 814 #endif 815 ; 816 } 817 818 static void ip_do_redirect(struct dst_entry *dst, struct sock *sk, struct sk_buff *skb) 819 { 820 struct rtable *rt; 821 struct flowi4 fl4; 822 const struct iphdr *iph = (const struct iphdr *) skb->data; 823 struct net *net = dev_net(skb->dev); 824 int oif = skb->dev->ifindex; 825 u8 tos = RT_TOS(iph->tos); 826 u8 prot = iph->protocol; 827 u32 mark = skb->mark; 828 829 rt = (struct rtable *) dst; 830 831 __build_flow_key(net, &fl4, sk, iph, oif, tos, prot, mark, 0); 832 __ip_do_redirect(rt, skb, &fl4, true); 833 } 834 835 static struct dst_entry *ipv4_negative_advice(struct dst_entry *dst) 836 { 837 struct rtable *rt = (struct rtable *)dst; 838 struct dst_entry *ret = dst; 839 840 if (rt) { 841 if (dst->obsolete > 0) { 842 ip_rt_put(rt); 843 ret = NULL; 844 } else if ((rt->rt_flags & RTCF_REDIRECTED) || 845 rt->dst.expires) { 846 ip_rt_put(rt); 847 ret = NULL; 848 } 849 } 850 return ret; 851 } 852 853 /* 854 * Algorithm: 855 * 1. The first ip_rt_redirect_number redirects are sent 856 * with exponential backoff, then we stop sending them at all, 857 * assuming that the host ignores our redirects. 858 * 2. If we did not see packets requiring redirects 859 * during ip_rt_redirect_silence, we assume that the host 860 * forgot redirected route and start to send redirects again. 861 * 862 * This algorithm is much cheaper and more intelligent than dumb load limiting 863 * in icmp.c. 864 * 865 * NOTE. Do not forget to inhibit load limiting for redirects (redundant) 866 * and "frag. need" (breaks PMTU discovery) in icmp.c. 867 */ 868 869 void ip_rt_send_redirect(struct sk_buff *skb) 870 { 871 struct rtable *rt = skb_rtable(skb); 872 struct in_device *in_dev; 873 struct inet_peer *peer; 874 struct net *net; 875 int log_martians; 876 int vif; 877 878 rcu_read_lock(); 879 in_dev = __in_dev_get_rcu(rt->dst.dev); 880 if (!in_dev || !IN_DEV_TX_REDIRECTS(in_dev)) { 881 rcu_read_unlock(); 882 return; 883 } 884 log_martians = IN_DEV_LOG_MARTIANS(in_dev); 885 vif = l3mdev_master_ifindex_rcu(rt->dst.dev); 886 rcu_read_unlock(); 887 888 net = dev_net(rt->dst.dev); 889 peer = inet_getpeer_v4(net->ipv4.peers, ip_hdr(skb)->saddr, vif, 1); 890 if (!peer) { 891 icmp_send(skb, ICMP_REDIRECT, ICMP_REDIR_HOST, 892 rt_nexthop(rt, ip_hdr(skb)->daddr)); 893 return; 894 } 895 896 /* No redirected packets during ip_rt_redirect_silence; 897 * reset the algorithm. 898 */ 899 if (time_after(jiffies, peer->rate_last + ip_rt_redirect_silence)) 900 peer->rate_tokens = 0; 901 902 /* Too many ignored redirects; do not send anything 903 * set dst.rate_last to the last seen redirected packet. 904 */ 905 if (peer->rate_tokens >= ip_rt_redirect_number) { 906 peer->rate_last = jiffies; 907 goto out_put_peer; 908 } 909 910 /* Check for load limit; set rate_last to the latest sent 911 * redirect. 912 */ 913 if (peer->rate_tokens == 0 || 914 time_after(jiffies, 915 (peer->rate_last + 916 (ip_rt_redirect_load << peer->rate_tokens)))) { 917 __be32 gw = rt_nexthop(rt, ip_hdr(skb)->daddr); 918 919 icmp_send(skb, ICMP_REDIRECT, ICMP_REDIR_HOST, gw); 920 peer->rate_last = jiffies; 921 ++peer->rate_tokens; 922 #ifdef CONFIG_IP_ROUTE_VERBOSE 923 if (log_martians && 924 peer->rate_tokens == ip_rt_redirect_number) 925 net_warn_ratelimited("host %pI4/if%d ignores redirects for %pI4 to %pI4\n", 926 &ip_hdr(skb)->saddr, inet_iif(skb), 927 &ip_hdr(skb)->daddr, &gw); 928 #endif 929 } 930 out_put_peer: 931 inet_putpeer(peer); 932 } 933 934 static int ip_error(struct sk_buff *skb) 935 { 936 struct in_device *in_dev = __in_dev_get_rcu(skb->dev); 937 struct rtable *rt = skb_rtable(skb); 938 struct inet_peer *peer; 939 unsigned long now; 940 struct net *net; 941 bool send; 942 int code; 943 944 /* IP on this device is disabled. */ 945 if (!in_dev) 946 goto out; 947 948 net = dev_net(rt->dst.dev); 949 if (!IN_DEV_FORWARD(in_dev)) { 950 switch (rt->dst.error) { 951 case EHOSTUNREACH: 952 __IP_INC_STATS(net, IPSTATS_MIB_INADDRERRORS); 953 break; 954 955 case ENETUNREACH: 956 __IP_INC_STATS(net, IPSTATS_MIB_INNOROUTES); 957 break; 958 } 959 goto out; 960 } 961 962 switch (rt->dst.error) { 963 case EINVAL: 964 default: 965 goto out; 966 case EHOSTUNREACH: 967 code = ICMP_HOST_UNREACH; 968 break; 969 case ENETUNREACH: 970 code = ICMP_NET_UNREACH; 971 __IP_INC_STATS(net, IPSTATS_MIB_INNOROUTES); 972 break; 973 case EACCES: 974 code = ICMP_PKT_FILTERED; 975 break; 976 } 977 978 peer = inet_getpeer_v4(net->ipv4.peers, ip_hdr(skb)->saddr, 979 l3mdev_master_ifindex(skb->dev), 1); 980 981 send = true; 982 if (peer) { 983 now = jiffies; 984 peer->rate_tokens += now - peer->rate_last; 985 if (peer->rate_tokens > ip_rt_error_burst) 986 peer->rate_tokens = ip_rt_error_burst; 987 peer->rate_last = now; 988 if (peer->rate_tokens >= ip_rt_error_cost) 989 peer->rate_tokens -= ip_rt_error_cost; 990 else 991 send = false; 992 inet_putpeer(peer); 993 } 994 if (send) 995 icmp_send(skb, ICMP_DEST_UNREACH, code, 0); 996 997 out: kfree_skb(skb); 998 return 0; 999 } 1000 1001 static void __ip_rt_update_pmtu(struct rtable *rt, struct flowi4 *fl4, u32 mtu) 1002 { 1003 struct dst_entry *dst = &rt->dst; 1004 struct fib_result res; 1005 1006 if (dst_metric_locked(dst, RTAX_MTU)) 1007 return; 1008 1009 if (ipv4_mtu(dst) < mtu) 1010 return; 1011 1012 if (mtu < ip_rt_min_pmtu) 1013 mtu = ip_rt_min_pmtu; 1014 1015 if (rt->rt_pmtu == mtu && 1016 time_before(jiffies, dst->expires - ip_rt_mtu_expires / 2)) 1017 return; 1018 1019 rcu_read_lock(); 1020 if (fib_lookup(dev_net(dst->dev), fl4, &res, 0) == 0) { 1021 struct fib_nh *nh = &FIB_RES_NH(res); 1022 1023 update_or_create_fnhe(nh, fl4->daddr, 0, mtu, 1024 jiffies + ip_rt_mtu_expires); 1025 } 1026 rcu_read_unlock(); 1027 } 1028 1029 static void ip_rt_update_pmtu(struct dst_entry *dst, struct sock *sk, 1030 struct sk_buff *skb, u32 mtu) 1031 { 1032 struct rtable *rt = (struct rtable *) dst; 1033 struct flowi4 fl4; 1034 1035 ip_rt_build_flow_key(&fl4, sk, skb); 1036 __ip_rt_update_pmtu(rt, &fl4, mtu); 1037 } 1038 1039 void ipv4_update_pmtu(struct sk_buff *skb, struct net *net, u32 mtu, 1040 int oif, u32 mark, u8 protocol, int flow_flags) 1041 { 1042 const struct iphdr *iph = (const struct iphdr *) skb->data; 1043 struct flowi4 fl4; 1044 struct rtable *rt; 1045 1046 if (!mark) 1047 mark = IP4_REPLY_MARK(net, skb->mark); 1048 1049 __build_flow_key(net, &fl4, NULL, iph, oif, 1050 RT_TOS(iph->tos), protocol, mark, flow_flags); 1051 rt = __ip_route_output_key(net, &fl4); 1052 if (!IS_ERR(rt)) { 1053 __ip_rt_update_pmtu(rt, &fl4, mtu); 1054 ip_rt_put(rt); 1055 } 1056 } 1057 EXPORT_SYMBOL_GPL(ipv4_update_pmtu); 1058 1059 static void __ipv4_sk_update_pmtu(struct sk_buff *skb, struct sock *sk, u32 mtu) 1060 { 1061 const struct iphdr *iph = (const struct iphdr *) skb->data; 1062 struct flowi4 fl4; 1063 struct rtable *rt; 1064 1065 __build_flow_key(sock_net(sk), &fl4, sk, iph, 0, 0, 0, 0, 0); 1066 1067 if (!fl4.flowi4_mark) 1068 fl4.flowi4_mark = IP4_REPLY_MARK(sock_net(sk), skb->mark); 1069 1070 rt = __ip_route_output_key(sock_net(sk), &fl4); 1071 if (!IS_ERR(rt)) { 1072 __ip_rt_update_pmtu(rt, &fl4, mtu); 1073 ip_rt_put(rt); 1074 } 1075 } 1076 1077 void ipv4_sk_update_pmtu(struct sk_buff *skb, struct sock *sk, u32 mtu) 1078 { 1079 const struct iphdr *iph = (const struct iphdr *) skb->data; 1080 struct flowi4 fl4; 1081 struct rtable *rt; 1082 struct dst_entry *odst = NULL; 1083 bool new = false; 1084 struct net *net = sock_net(sk); 1085 1086 bh_lock_sock(sk); 1087 1088 if (!ip_sk_accept_pmtu(sk)) 1089 goto out; 1090 1091 odst = sk_dst_get(sk); 1092 1093 if (sock_owned_by_user(sk) || !odst) { 1094 __ipv4_sk_update_pmtu(skb, sk, mtu); 1095 goto out; 1096 } 1097 1098 __build_flow_key(net, &fl4, sk, iph, 0, 0, 0, 0, 0); 1099 1100 rt = (struct rtable *)odst; 1101 if (odst->obsolete && !odst->ops->check(odst, 0)) { 1102 rt = ip_route_output_flow(sock_net(sk), &fl4, sk); 1103 if (IS_ERR(rt)) 1104 goto out; 1105 1106 new = true; 1107 } 1108 1109 __ip_rt_update_pmtu((struct rtable *) rt->dst.path, &fl4, mtu); 1110 1111 if (!dst_check(&rt->dst, 0)) { 1112 if (new) 1113 dst_release(&rt->dst); 1114 1115 rt = ip_route_output_flow(sock_net(sk), &fl4, sk); 1116 if (IS_ERR(rt)) 1117 goto out; 1118 1119 new = true; 1120 } 1121 1122 if (new) 1123 sk_dst_set(sk, &rt->dst); 1124 1125 out: 1126 bh_unlock_sock(sk); 1127 dst_release(odst); 1128 } 1129 EXPORT_SYMBOL_GPL(ipv4_sk_update_pmtu); 1130 1131 void ipv4_redirect(struct sk_buff *skb, struct net *net, 1132 int oif, u32 mark, u8 protocol, int flow_flags) 1133 { 1134 const struct iphdr *iph = (const struct iphdr *) skb->data; 1135 struct flowi4 fl4; 1136 struct rtable *rt; 1137 1138 __build_flow_key(net, &fl4, NULL, iph, oif, 1139 RT_TOS(iph->tos), protocol, mark, flow_flags); 1140 rt = __ip_route_output_key(net, &fl4); 1141 if (!IS_ERR(rt)) { 1142 __ip_do_redirect(rt, skb, &fl4, false); 1143 ip_rt_put(rt); 1144 } 1145 } 1146 EXPORT_SYMBOL_GPL(ipv4_redirect); 1147 1148 void ipv4_sk_redirect(struct sk_buff *skb, struct sock *sk) 1149 { 1150 const struct iphdr *iph = (const struct iphdr *) skb->data; 1151 struct flowi4 fl4; 1152 struct rtable *rt; 1153 struct net *net = sock_net(sk); 1154 1155 __build_flow_key(net, &fl4, sk, iph, 0, 0, 0, 0, 0); 1156 rt = __ip_route_output_key(net, &fl4); 1157 if (!IS_ERR(rt)) { 1158 __ip_do_redirect(rt, skb, &fl4, false); 1159 ip_rt_put(rt); 1160 } 1161 } 1162 EXPORT_SYMBOL_GPL(ipv4_sk_redirect); 1163 1164 static struct dst_entry *ipv4_dst_check(struct dst_entry *dst, u32 cookie) 1165 { 1166 struct rtable *rt = (struct rtable *) dst; 1167 1168 /* All IPV4 dsts are created with ->obsolete set to the value 1169 * DST_OBSOLETE_FORCE_CHK which forces validation calls down 1170 * into this function always. 1171 * 1172 * When a PMTU/redirect information update invalidates a route, 1173 * this is indicated by setting obsolete to DST_OBSOLETE_KILL or 1174 * DST_OBSOLETE_DEAD by dst_free(). 1175 */ 1176 if (dst->obsolete != DST_OBSOLETE_FORCE_CHK || rt_is_expired(rt)) 1177 return NULL; 1178 return dst; 1179 } 1180 1181 static void ipv4_link_failure(struct sk_buff *skb) 1182 { 1183 struct rtable *rt; 1184 1185 icmp_send(skb, ICMP_DEST_UNREACH, ICMP_HOST_UNREACH, 0); 1186 1187 rt = skb_rtable(skb); 1188 if (rt) 1189 dst_set_expires(&rt->dst, 0); 1190 } 1191 1192 static int ip_rt_bug(struct net *net, struct sock *sk, struct sk_buff *skb) 1193 { 1194 pr_debug("%s: %pI4 -> %pI4, %s\n", 1195 __func__, &ip_hdr(skb)->saddr, &ip_hdr(skb)->daddr, 1196 skb->dev ? skb->dev->name : "?"); 1197 kfree_skb(skb); 1198 WARN_ON(1); 1199 return 0; 1200 } 1201 1202 /* 1203 We do not cache source address of outgoing interface, 1204 because it is used only by IP RR, TS and SRR options, 1205 so that it out of fast path. 1206 1207 BTW remember: "addr" is allowed to be not aligned 1208 in IP options! 1209 */ 1210 1211 void ip_rt_get_source(u8 *addr, struct sk_buff *skb, struct rtable *rt) 1212 { 1213 __be32 src; 1214 1215 if (rt_is_output_route(rt)) 1216 src = ip_hdr(skb)->saddr; 1217 else { 1218 struct fib_result res; 1219 struct flowi4 fl4; 1220 struct iphdr *iph; 1221 1222 iph = ip_hdr(skb); 1223 1224 memset(&fl4, 0, sizeof(fl4)); 1225 fl4.daddr = iph->daddr; 1226 fl4.saddr = iph->saddr; 1227 fl4.flowi4_tos = RT_TOS(iph->tos); 1228 fl4.flowi4_oif = rt->dst.dev->ifindex; 1229 fl4.flowi4_iif = skb->dev->ifindex; 1230 fl4.flowi4_mark = skb->mark; 1231 1232 rcu_read_lock(); 1233 if (fib_lookup(dev_net(rt->dst.dev), &fl4, &res, 0) == 0) 1234 src = FIB_RES_PREFSRC(dev_net(rt->dst.dev), res); 1235 else 1236 src = inet_select_addr(rt->dst.dev, 1237 rt_nexthop(rt, iph->daddr), 1238 RT_SCOPE_UNIVERSE); 1239 rcu_read_unlock(); 1240 } 1241 memcpy(addr, &src, 4); 1242 } 1243 1244 #ifdef CONFIG_IP_ROUTE_CLASSID 1245 static void set_class_tag(struct rtable *rt, u32 tag) 1246 { 1247 if (!(rt->dst.tclassid & 0xFFFF)) 1248 rt->dst.tclassid |= tag & 0xFFFF; 1249 if (!(rt->dst.tclassid & 0xFFFF0000)) 1250 rt->dst.tclassid |= tag & 0xFFFF0000; 1251 } 1252 #endif 1253 1254 static unsigned int ipv4_default_advmss(const struct dst_entry *dst) 1255 { 1256 unsigned int header_size = sizeof(struct tcphdr) + sizeof(struct iphdr); 1257 unsigned int advmss = max_t(unsigned int, ipv4_mtu(dst) - header_size, 1258 ip_rt_min_advmss); 1259 1260 return min(advmss, IPV4_MAX_PMTU - header_size); 1261 } 1262 1263 static unsigned int ipv4_mtu(const struct dst_entry *dst) 1264 { 1265 const struct rtable *rt = (const struct rtable *) dst; 1266 unsigned int mtu = rt->rt_pmtu; 1267 1268 if (!mtu || time_after_eq(jiffies, rt->dst.expires)) 1269 mtu = dst_metric_raw(dst, RTAX_MTU); 1270 1271 if (mtu) 1272 return mtu; 1273 1274 mtu = READ_ONCE(dst->dev->mtu); 1275 1276 if (unlikely(dst_metric_locked(dst, RTAX_MTU))) { 1277 if (rt->rt_uses_gateway && mtu > 576) 1278 mtu = 576; 1279 } 1280 1281 mtu = min_t(unsigned int, mtu, IP_MAX_MTU); 1282 1283 return mtu - lwtunnel_headroom(dst->lwtstate, mtu); 1284 } 1285 1286 static struct fib_nh_exception *find_exception(struct fib_nh *nh, __be32 daddr) 1287 { 1288 struct fnhe_hash_bucket *hash = rcu_dereference(nh->nh_exceptions); 1289 struct fib_nh_exception *fnhe; 1290 u32 hval; 1291 1292 if (!hash) 1293 return NULL; 1294 1295 hval = fnhe_hashfun(daddr); 1296 1297 for (fnhe = rcu_dereference(hash[hval].chain); fnhe; 1298 fnhe = rcu_dereference(fnhe->fnhe_next)) { 1299 if (fnhe->fnhe_daddr == daddr) 1300 return fnhe; 1301 } 1302 return NULL; 1303 } 1304 1305 static bool rt_bind_exception(struct rtable *rt, struct fib_nh_exception *fnhe, 1306 __be32 daddr, const bool do_cache) 1307 { 1308 bool ret = false; 1309 1310 spin_lock_bh(&fnhe_lock); 1311 1312 if (daddr == fnhe->fnhe_daddr) { 1313 struct rtable __rcu **porig; 1314 struct rtable *orig; 1315 int genid = fnhe_genid(dev_net(rt->dst.dev)); 1316 1317 if (rt_is_input_route(rt)) 1318 porig = &fnhe->fnhe_rth_input; 1319 else 1320 porig = &fnhe->fnhe_rth_output; 1321 orig = rcu_dereference(*porig); 1322 1323 if (fnhe->fnhe_genid != genid) { 1324 fnhe->fnhe_genid = genid; 1325 fnhe->fnhe_gw = 0; 1326 fnhe->fnhe_pmtu = 0; 1327 fnhe->fnhe_expires = 0; 1328 fnhe_flush_routes(fnhe); 1329 orig = NULL; 1330 } 1331 fill_route_from_fnhe(rt, fnhe); 1332 if (!rt->rt_gateway) 1333 rt->rt_gateway = daddr; 1334 1335 if (do_cache) { 1336 dst_hold(&rt->dst); 1337 rcu_assign_pointer(*porig, rt); 1338 if (orig) { 1339 dst_dev_put(&orig->dst); 1340 dst_release(&orig->dst); 1341 } 1342 ret = true; 1343 } 1344 1345 fnhe->fnhe_stamp = jiffies; 1346 } 1347 spin_unlock_bh(&fnhe_lock); 1348 1349 return ret; 1350 } 1351 1352 static bool rt_cache_route(struct fib_nh *nh, struct rtable *rt) 1353 { 1354 struct rtable *orig, *prev, **p; 1355 bool ret = true; 1356 1357 if (rt_is_input_route(rt)) { 1358 p = (struct rtable **)&nh->nh_rth_input; 1359 } else { 1360 p = (struct rtable **)raw_cpu_ptr(nh->nh_pcpu_rth_output); 1361 } 1362 orig = *p; 1363 1364 /* hold dst before doing cmpxchg() to avoid race condition 1365 * on this dst 1366 */ 1367 dst_hold(&rt->dst); 1368 prev = cmpxchg(p, orig, rt); 1369 if (prev == orig) { 1370 if (orig) { 1371 dst_dev_put(&orig->dst); 1372 dst_release(&orig->dst); 1373 } 1374 } else { 1375 dst_release(&rt->dst); 1376 ret = false; 1377 } 1378 1379 return ret; 1380 } 1381 1382 struct uncached_list { 1383 spinlock_t lock; 1384 struct list_head head; 1385 }; 1386 1387 static DEFINE_PER_CPU_ALIGNED(struct uncached_list, rt_uncached_list); 1388 1389 static void rt_add_uncached_list(struct rtable *rt) 1390 { 1391 struct uncached_list *ul = raw_cpu_ptr(&rt_uncached_list); 1392 1393 rt->rt_uncached_list = ul; 1394 1395 spin_lock_bh(&ul->lock); 1396 list_add_tail(&rt->rt_uncached, &ul->head); 1397 spin_unlock_bh(&ul->lock); 1398 } 1399 1400 static void ipv4_dst_destroy(struct dst_entry *dst) 1401 { 1402 struct dst_metrics *p = (struct dst_metrics *)DST_METRICS_PTR(dst); 1403 struct rtable *rt = (struct rtable *) dst; 1404 1405 if (p != &dst_default_metrics && refcount_dec_and_test(&p->refcnt)) 1406 kfree(p); 1407 1408 if (!list_empty(&rt->rt_uncached)) { 1409 struct uncached_list *ul = rt->rt_uncached_list; 1410 1411 spin_lock_bh(&ul->lock); 1412 list_del(&rt->rt_uncached); 1413 spin_unlock_bh(&ul->lock); 1414 } 1415 } 1416 1417 void rt_flush_dev(struct net_device *dev) 1418 { 1419 struct net *net = dev_net(dev); 1420 struct rtable *rt; 1421 int cpu; 1422 1423 for_each_possible_cpu(cpu) { 1424 struct uncached_list *ul = &per_cpu(rt_uncached_list, cpu); 1425 1426 spin_lock_bh(&ul->lock); 1427 list_for_each_entry(rt, &ul->head, rt_uncached) { 1428 if (rt->dst.dev != dev) 1429 continue; 1430 rt->dst.dev = net->loopback_dev; 1431 dev_hold(rt->dst.dev); 1432 dev_put(dev); 1433 } 1434 spin_unlock_bh(&ul->lock); 1435 } 1436 } 1437 1438 static bool rt_cache_valid(const struct rtable *rt) 1439 { 1440 return rt && 1441 rt->dst.obsolete == DST_OBSOLETE_FORCE_CHK && 1442 !rt_is_expired(rt); 1443 } 1444 1445 static void rt_set_nexthop(struct rtable *rt, __be32 daddr, 1446 const struct fib_result *res, 1447 struct fib_nh_exception *fnhe, 1448 struct fib_info *fi, u16 type, u32 itag, 1449 const bool do_cache) 1450 { 1451 bool cached = false; 1452 1453 if (fi) { 1454 struct fib_nh *nh = &FIB_RES_NH(*res); 1455 1456 if (nh->nh_gw && nh->nh_scope == RT_SCOPE_LINK) { 1457 rt->rt_gateway = nh->nh_gw; 1458 rt->rt_uses_gateway = 1; 1459 } 1460 dst_init_metrics(&rt->dst, fi->fib_metrics->metrics, true); 1461 if (fi->fib_metrics != &dst_default_metrics) { 1462 rt->dst._metrics |= DST_METRICS_REFCOUNTED; 1463 refcount_inc(&fi->fib_metrics->refcnt); 1464 } 1465 #ifdef CONFIG_IP_ROUTE_CLASSID 1466 rt->dst.tclassid = nh->nh_tclassid; 1467 #endif 1468 rt->dst.lwtstate = lwtstate_get(nh->nh_lwtstate); 1469 if (unlikely(fnhe)) 1470 cached = rt_bind_exception(rt, fnhe, daddr, do_cache); 1471 else if (do_cache) 1472 cached = rt_cache_route(nh, rt); 1473 if (unlikely(!cached)) { 1474 /* Routes we intend to cache in nexthop exception or 1475 * FIB nexthop have the DST_NOCACHE bit clear. 1476 * However, if we are unsuccessful at storing this 1477 * route into the cache we really need to set it. 1478 */ 1479 if (!rt->rt_gateway) 1480 rt->rt_gateway = daddr; 1481 rt_add_uncached_list(rt); 1482 } 1483 } else 1484 rt_add_uncached_list(rt); 1485 1486 #ifdef CONFIG_IP_ROUTE_CLASSID 1487 #ifdef CONFIG_IP_MULTIPLE_TABLES 1488 set_class_tag(rt, res->tclassid); 1489 #endif 1490 set_class_tag(rt, itag); 1491 #endif 1492 } 1493 1494 struct rtable *rt_dst_alloc(struct net_device *dev, 1495 unsigned int flags, u16 type, 1496 bool nopolicy, bool noxfrm, bool will_cache) 1497 { 1498 struct rtable *rt; 1499 1500 rt = dst_alloc(&ipv4_dst_ops, dev, 1, DST_OBSOLETE_FORCE_CHK, 1501 (will_cache ? 0 : DST_HOST) | 1502 (nopolicy ? DST_NOPOLICY : 0) | 1503 (noxfrm ? DST_NOXFRM : 0)); 1504 1505 if (rt) { 1506 rt->rt_genid = rt_genid_ipv4(dev_net(dev)); 1507 rt->rt_flags = flags; 1508 rt->rt_type = type; 1509 rt->rt_is_input = 0; 1510 rt->rt_iif = 0; 1511 rt->rt_pmtu = 0; 1512 rt->rt_gateway = 0; 1513 rt->rt_uses_gateway = 0; 1514 rt->rt_table_id = 0; 1515 INIT_LIST_HEAD(&rt->rt_uncached); 1516 1517 rt->dst.output = ip_output; 1518 if (flags & RTCF_LOCAL) 1519 rt->dst.input = ip_local_deliver; 1520 } 1521 1522 return rt; 1523 } 1524 EXPORT_SYMBOL(rt_dst_alloc); 1525 1526 /* called in rcu_read_lock() section */ 1527 int ip_mc_validate_source(struct sk_buff *skb, __be32 daddr, __be32 saddr, 1528 u8 tos, struct net_device *dev, 1529 struct in_device *in_dev, u32 *itag) 1530 { 1531 int err; 1532 1533 /* Primary sanity checks. */ 1534 if (!in_dev) 1535 return -EINVAL; 1536 1537 if (ipv4_is_multicast(saddr) || ipv4_is_lbcast(saddr) || 1538 skb->protocol != htons(ETH_P_IP)) 1539 return -EINVAL; 1540 1541 if (ipv4_is_loopback(saddr) && !IN_DEV_ROUTE_LOCALNET(in_dev)) 1542 return -EINVAL; 1543 1544 if (ipv4_is_zeronet(saddr)) { 1545 if (!ipv4_is_local_multicast(daddr)) 1546 return -EINVAL; 1547 } else { 1548 err = fib_validate_source(skb, saddr, 0, tos, 0, dev, 1549 in_dev, itag); 1550 if (err < 0) 1551 return err; 1552 } 1553 return 0; 1554 } 1555 1556 /* called in rcu_read_lock() section */ 1557 static int ip_route_input_mc(struct sk_buff *skb, __be32 daddr, __be32 saddr, 1558 u8 tos, struct net_device *dev, int our) 1559 { 1560 struct in_device *in_dev = __in_dev_get_rcu(dev); 1561 unsigned int flags = RTCF_MULTICAST; 1562 struct rtable *rth; 1563 u32 itag = 0; 1564 int err; 1565 1566 err = ip_mc_validate_source(skb, daddr, saddr, tos, dev, in_dev, &itag); 1567 if (err) 1568 return err; 1569 1570 if (our) 1571 flags |= RTCF_LOCAL; 1572 1573 rth = rt_dst_alloc(dev_net(dev)->loopback_dev, flags, RTN_MULTICAST, 1574 IN_DEV_CONF_GET(in_dev, NOPOLICY), false, false); 1575 if (!rth) 1576 return -ENOBUFS; 1577 1578 #ifdef CONFIG_IP_ROUTE_CLASSID 1579 rth->dst.tclassid = itag; 1580 #endif 1581 rth->dst.output = ip_rt_bug; 1582 rth->rt_is_input= 1; 1583 1584 #ifdef CONFIG_IP_MROUTE 1585 if (!ipv4_is_local_multicast(daddr) && IN_DEV_MFORWARD(in_dev)) 1586 rth->dst.input = ip_mr_input; 1587 #endif 1588 RT_CACHE_STAT_INC(in_slow_mc); 1589 1590 skb_dst_set(skb, &rth->dst); 1591 return 0; 1592 } 1593 1594 1595 static void ip_handle_martian_source(struct net_device *dev, 1596 struct in_device *in_dev, 1597 struct sk_buff *skb, 1598 __be32 daddr, 1599 __be32 saddr) 1600 { 1601 RT_CACHE_STAT_INC(in_martian_src); 1602 #ifdef CONFIG_IP_ROUTE_VERBOSE 1603 if (IN_DEV_LOG_MARTIANS(in_dev) && net_ratelimit()) { 1604 /* 1605 * RFC1812 recommendation, if source is martian, 1606 * the only hint is MAC header. 1607 */ 1608 pr_warn("martian source %pI4 from %pI4, on dev %s\n", 1609 &daddr, &saddr, dev->name); 1610 if (dev->hard_header_len && skb_mac_header_was_set(skb)) { 1611 print_hex_dump(KERN_WARNING, "ll header: ", 1612 DUMP_PREFIX_OFFSET, 16, 1, 1613 skb_mac_header(skb), 1614 dev->hard_header_len, true); 1615 } 1616 } 1617 #endif 1618 } 1619 1620 static void ip_del_fnhe(struct fib_nh *nh, __be32 daddr) 1621 { 1622 struct fnhe_hash_bucket *hash; 1623 struct fib_nh_exception *fnhe, __rcu **fnhe_p; 1624 u32 hval = fnhe_hashfun(daddr); 1625 1626 spin_lock_bh(&fnhe_lock); 1627 1628 hash = rcu_dereference_protected(nh->nh_exceptions, 1629 lockdep_is_held(&fnhe_lock)); 1630 hash += hval; 1631 1632 fnhe_p = &hash->chain; 1633 fnhe = rcu_dereference_protected(*fnhe_p, lockdep_is_held(&fnhe_lock)); 1634 while (fnhe) { 1635 if (fnhe->fnhe_daddr == daddr) { 1636 rcu_assign_pointer(*fnhe_p, rcu_dereference_protected( 1637 fnhe->fnhe_next, lockdep_is_held(&fnhe_lock))); 1638 fnhe_flush_routes(fnhe); 1639 kfree_rcu(fnhe, rcu); 1640 break; 1641 } 1642 fnhe_p = &fnhe->fnhe_next; 1643 fnhe = rcu_dereference_protected(fnhe->fnhe_next, 1644 lockdep_is_held(&fnhe_lock)); 1645 } 1646 1647 spin_unlock_bh(&fnhe_lock); 1648 } 1649 1650 static void set_lwt_redirect(struct rtable *rth) 1651 { 1652 if (lwtunnel_output_redirect(rth->dst.lwtstate)) { 1653 rth->dst.lwtstate->orig_output = rth->dst.output; 1654 rth->dst.output = lwtunnel_output; 1655 } 1656 1657 if (lwtunnel_input_redirect(rth->dst.lwtstate)) { 1658 rth->dst.lwtstate->orig_input = rth->dst.input; 1659 rth->dst.input = lwtunnel_input; 1660 } 1661 } 1662 1663 /* called in rcu_read_lock() section */ 1664 static int __mkroute_input(struct sk_buff *skb, 1665 const struct fib_result *res, 1666 struct in_device *in_dev, 1667 __be32 daddr, __be32 saddr, u32 tos) 1668 { 1669 struct fib_nh_exception *fnhe; 1670 struct rtable *rth; 1671 int err; 1672 struct in_device *out_dev; 1673 bool do_cache; 1674 u32 itag = 0; 1675 1676 /* get a working reference to the output device */ 1677 out_dev = __in_dev_get_rcu(FIB_RES_DEV(*res)); 1678 if (!out_dev) { 1679 net_crit_ratelimited("Bug in ip_route_input_slow(). Please report.\n"); 1680 return -EINVAL; 1681 } 1682 1683 err = fib_validate_source(skb, saddr, daddr, tos, FIB_RES_OIF(*res), 1684 in_dev->dev, in_dev, &itag); 1685 if (err < 0) { 1686 ip_handle_martian_source(in_dev->dev, in_dev, skb, daddr, 1687 saddr); 1688 1689 goto cleanup; 1690 } 1691 1692 do_cache = res->fi && !itag; 1693 if (out_dev == in_dev && err && IN_DEV_TX_REDIRECTS(out_dev) && 1694 skb->protocol == htons(ETH_P_IP) && 1695 (IN_DEV_SHARED_MEDIA(out_dev) || 1696 inet_addr_onlink(out_dev, saddr, FIB_RES_GW(*res)))) 1697 IPCB(skb)->flags |= IPSKB_DOREDIRECT; 1698 1699 if (skb->protocol != htons(ETH_P_IP)) { 1700 /* Not IP (i.e. ARP). Do not create route, if it is 1701 * invalid for proxy arp. DNAT routes are always valid. 1702 * 1703 * Proxy arp feature have been extended to allow, ARP 1704 * replies back to the same interface, to support 1705 * Private VLAN switch technologies. See arp.c. 1706 */ 1707 if (out_dev == in_dev && 1708 IN_DEV_PROXY_ARP_PVLAN(in_dev) == 0) { 1709 err = -EINVAL; 1710 goto cleanup; 1711 } 1712 } 1713 1714 fnhe = find_exception(&FIB_RES_NH(*res), daddr); 1715 if (do_cache) { 1716 if (fnhe) { 1717 rth = rcu_dereference(fnhe->fnhe_rth_input); 1718 if (rth && rth->dst.expires && 1719 time_after(jiffies, rth->dst.expires)) { 1720 ip_del_fnhe(&FIB_RES_NH(*res), daddr); 1721 fnhe = NULL; 1722 } else { 1723 goto rt_cache; 1724 } 1725 } 1726 1727 rth = rcu_dereference(FIB_RES_NH(*res).nh_rth_input); 1728 1729 rt_cache: 1730 if (rt_cache_valid(rth)) { 1731 skb_dst_set_noref(skb, &rth->dst); 1732 goto out; 1733 } 1734 } 1735 1736 rth = rt_dst_alloc(out_dev->dev, 0, res->type, 1737 IN_DEV_CONF_GET(in_dev, NOPOLICY), 1738 IN_DEV_CONF_GET(out_dev, NOXFRM), do_cache); 1739 if (!rth) { 1740 err = -ENOBUFS; 1741 goto cleanup; 1742 } 1743 1744 rth->rt_is_input = 1; 1745 if (res->table) 1746 rth->rt_table_id = res->table->tb_id; 1747 RT_CACHE_STAT_INC(in_slow_tot); 1748 1749 rth->dst.input = ip_forward; 1750 1751 rt_set_nexthop(rth, daddr, res, fnhe, res->fi, res->type, itag, 1752 do_cache); 1753 set_lwt_redirect(rth); 1754 skb_dst_set(skb, &rth->dst); 1755 out: 1756 err = 0; 1757 cleanup: 1758 return err; 1759 } 1760 1761 #ifdef CONFIG_IP_ROUTE_MULTIPATH 1762 /* To make ICMP packets follow the right flow, the multipath hash is 1763 * calculated from the inner IP addresses. 1764 */ 1765 static void ip_multipath_l3_keys(const struct sk_buff *skb, 1766 struct flow_keys *hash_keys) 1767 { 1768 const struct iphdr *outer_iph = ip_hdr(skb); 1769 const struct iphdr *inner_iph; 1770 const struct icmphdr *icmph; 1771 struct iphdr _inner_iph; 1772 struct icmphdr _icmph; 1773 1774 hash_keys->addrs.v4addrs.src = outer_iph->saddr; 1775 hash_keys->addrs.v4addrs.dst = outer_iph->daddr; 1776 if (likely(outer_iph->protocol != IPPROTO_ICMP)) 1777 return; 1778 1779 if (unlikely((outer_iph->frag_off & htons(IP_OFFSET)) != 0)) 1780 return; 1781 1782 icmph = skb_header_pointer(skb, outer_iph->ihl * 4, sizeof(_icmph), 1783 &_icmph); 1784 if (!icmph) 1785 return; 1786 1787 if (icmph->type != ICMP_DEST_UNREACH && 1788 icmph->type != ICMP_REDIRECT && 1789 icmph->type != ICMP_TIME_EXCEEDED && 1790 icmph->type != ICMP_PARAMETERPROB) 1791 return; 1792 1793 inner_iph = skb_header_pointer(skb, 1794 outer_iph->ihl * 4 + sizeof(_icmph), 1795 sizeof(_inner_iph), &_inner_iph); 1796 if (!inner_iph) 1797 return; 1798 hash_keys->addrs.v4addrs.src = inner_iph->saddr; 1799 hash_keys->addrs.v4addrs.dst = inner_iph->daddr; 1800 } 1801 1802 /* if skb is set it will be used and fl4 can be NULL */ 1803 int fib_multipath_hash(const struct fib_info *fi, const struct flowi4 *fl4, 1804 const struct sk_buff *skb) 1805 { 1806 struct net *net = fi->fib_net; 1807 struct flow_keys hash_keys; 1808 u32 mhash; 1809 1810 switch (net->ipv4.sysctl_fib_multipath_hash_policy) { 1811 case 0: 1812 memset(&hash_keys, 0, sizeof(hash_keys)); 1813 hash_keys.control.addr_type = FLOW_DISSECTOR_KEY_IPV4_ADDRS; 1814 if (skb) { 1815 ip_multipath_l3_keys(skb, &hash_keys); 1816 } else { 1817 hash_keys.addrs.v4addrs.src = fl4->saddr; 1818 hash_keys.addrs.v4addrs.dst = fl4->daddr; 1819 } 1820 break; 1821 case 1: 1822 /* skb is currently provided only when forwarding */ 1823 if (skb) { 1824 unsigned int flag = FLOW_DISSECTOR_F_STOP_AT_ENCAP; 1825 struct flow_keys keys; 1826 1827 /* short-circuit if we already have L4 hash present */ 1828 if (skb->l4_hash) 1829 return skb_get_hash_raw(skb) >> 1; 1830 memset(&hash_keys, 0, sizeof(hash_keys)); 1831 skb_flow_dissect_flow_keys(skb, &keys, flag); 1832 hash_keys.addrs.v4addrs.src = keys.addrs.v4addrs.src; 1833 hash_keys.addrs.v4addrs.dst = keys.addrs.v4addrs.dst; 1834 hash_keys.ports.src = keys.ports.src; 1835 hash_keys.ports.dst = keys.ports.dst; 1836 hash_keys.basic.ip_proto = keys.basic.ip_proto; 1837 } else { 1838 memset(&hash_keys, 0, sizeof(hash_keys)); 1839 hash_keys.control.addr_type = FLOW_DISSECTOR_KEY_IPV4_ADDRS; 1840 hash_keys.addrs.v4addrs.src = fl4->saddr; 1841 hash_keys.addrs.v4addrs.dst = fl4->daddr; 1842 hash_keys.ports.src = fl4->fl4_sport; 1843 hash_keys.ports.dst = fl4->fl4_dport; 1844 hash_keys.basic.ip_proto = fl4->flowi4_proto; 1845 } 1846 break; 1847 } 1848 mhash = flow_hash_from_keys(&hash_keys); 1849 1850 return mhash >> 1; 1851 } 1852 EXPORT_SYMBOL_GPL(fib_multipath_hash); 1853 #endif /* CONFIG_IP_ROUTE_MULTIPATH */ 1854 1855 static int ip_mkroute_input(struct sk_buff *skb, 1856 struct fib_result *res, 1857 struct in_device *in_dev, 1858 __be32 daddr, __be32 saddr, u32 tos) 1859 { 1860 #ifdef CONFIG_IP_ROUTE_MULTIPATH 1861 if (res->fi && res->fi->fib_nhs > 1) { 1862 int h = fib_multipath_hash(res->fi, NULL, skb); 1863 1864 fib_select_multipath(res, h); 1865 } 1866 #endif 1867 1868 /* create a routing cache entry */ 1869 return __mkroute_input(skb, res, in_dev, daddr, saddr, tos); 1870 } 1871 1872 /* 1873 * NOTE. We drop all the packets that has local source 1874 * addresses, because every properly looped back packet 1875 * must have correct destination already attached by output routine. 1876 * 1877 * Such approach solves two big problems: 1878 * 1. Not simplex devices are handled properly. 1879 * 2. IP spoofing attempts are filtered with 100% of guarantee. 1880 * called with rcu_read_lock() 1881 */ 1882 1883 static int ip_route_input_slow(struct sk_buff *skb, __be32 daddr, __be32 saddr, 1884 u8 tos, struct net_device *dev, 1885 struct fib_result *res) 1886 { 1887 struct in_device *in_dev = __in_dev_get_rcu(dev); 1888 struct ip_tunnel_info *tun_info; 1889 struct flowi4 fl4; 1890 unsigned int flags = 0; 1891 u32 itag = 0; 1892 struct rtable *rth; 1893 int err = -EINVAL; 1894 struct net *net = dev_net(dev); 1895 bool do_cache; 1896 1897 /* IP on this device is disabled. */ 1898 1899 if (!in_dev) 1900 goto out; 1901 1902 /* Check for the most weird martians, which can be not detected 1903 by fib_lookup. 1904 */ 1905 1906 tun_info = skb_tunnel_info(skb); 1907 if (tun_info && !(tun_info->mode & IP_TUNNEL_INFO_TX)) 1908 fl4.flowi4_tun_key.tun_id = tun_info->key.tun_id; 1909 else 1910 fl4.flowi4_tun_key.tun_id = 0; 1911 skb_dst_drop(skb); 1912 1913 if (ipv4_is_multicast(saddr) || ipv4_is_lbcast(saddr)) 1914 goto martian_source; 1915 1916 res->fi = NULL; 1917 res->table = NULL; 1918 if (ipv4_is_lbcast(daddr) || (saddr == 0 && daddr == 0)) 1919 goto brd_input; 1920 1921 /* Accept zero addresses only to limited broadcast; 1922 * I even do not know to fix it or not. Waiting for complains :-) 1923 */ 1924 if (ipv4_is_zeronet(saddr)) 1925 goto martian_source; 1926 1927 if (ipv4_is_zeronet(daddr)) 1928 goto martian_destination; 1929 1930 /* Following code try to avoid calling IN_DEV_NET_ROUTE_LOCALNET(), 1931 * and call it once if daddr or/and saddr are loopback addresses 1932 */ 1933 if (ipv4_is_loopback(daddr)) { 1934 if (!IN_DEV_NET_ROUTE_LOCALNET(in_dev, net)) 1935 goto martian_destination; 1936 } else if (ipv4_is_loopback(saddr)) { 1937 if (!IN_DEV_NET_ROUTE_LOCALNET(in_dev, net)) 1938 goto martian_source; 1939 } 1940 1941 /* 1942 * Now we are ready to route packet. 1943 */ 1944 fl4.flowi4_oif = 0; 1945 fl4.flowi4_iif = dev->ifindex; 1946 fl4.flowi4_mark = skb->mark; 1947 fl4.flowi4_tos = tos; 1948 fl4.flowi4_scope = RT_SCOPE_UNIVERSE; 1949 fl4.flowi4_flags = 0; 1950 fl4.daddr = daddr; 1951 fl4.saddr = saddr; 1952 fl4.flowi4_uid = sock_net_uid(net, NULL); 1953 err = fib_lookup(net, &fl4, res, 0); 1954 if (err != 0) { 1955 if (!IN_DEV_FORWARD(in_dev)) 1956 err = -EHOSTUNREACH; 1957 goto no_route; 1958 } 1959 1960 if (res->type == RTN_BROADCAST) 1961 goto brd_input; 1962 1963 if (res->type == RTN_LOCAL) { 1964 err = fib_validate_source(skb, saddr, daddr, tos, 1965 0, dev, in_dev, &itag); 1966 if (err < 0) 1967 goto martian_source; 1968 goto local_input; 1969 } 1970 1971 if (!IN_DEV_FORWARD(in_dev)) { 1972 err = -EHOSTUNREACH; 1973 goto no_route; 1974 } 1975 if (res->type != RTN_UNICAST) 1976 goto martian_destination; 1977 1978 err = ip_mkroute_input(skb, res, in_dev, daddr, saddr, tos); 1979 out: return err; 1980 1981 brd_input: 1982 if (skb->protocol != htons(ETH_P_IP)) 1983 goto e_inval; 1984 1985 if (!ipv4_is_zeronet(saddr)) { 1986 err = fib_validate_source(skb, saddr, 0, tos, 0, dev, 1987 in_dev, &itag); 1988 if (err < 0) 1989 goto martian_source; 1990 } 1991 flags |= RTCF_BROADCAST; 1992 res->type = RTN_BROADCAST; 1993 RT_CACHE_STAT_INC(in_brd); 1994 1995 local_input: 1996 do_cache = false; 1997 if (res->fi) { 1998 if (!itag) { 1999 rth = rcu_dereference(FIB_RES_NH(*res).nh_rth_input); 2000 if (rt_cache_valid(rth)) { 2001 skb_dst_set_noref(skb, &rth->dst); 2002 err = 0; 2003 goto out; 2004 } 2005 do_cache = true; 2006 } 2007 } 2008 2009 rth = rt_dst_alloc(l3mdev_master_dev_rcu(dev) ? : net->loopback_dev, 2010 flags | RTCF_LOCAL, res->type, 2011 IN_DEV_CONF_GET(in_dev, NOPOLICY), false, do_cache); 2012 if (!rth) 2013 goto e_nobufs; 2014 2015 rth->dst.output= ip_rt_bug; 2016 #ifdef CONFIG_IP_ROUTE_CLASSID 2017 rth->dst.tclassid = itag; 2018 #endif 2019 rth->rt_is_input = 1; 2020 if (res->table) 2021 rth->rt_table_id = res->table->tb_id; 2022 2023 RT_CACHE_STAT_INC(in_slow_tot); 2024 if (res->type == RTN_UNREACHABLE) { 2025 rth->dst.input= ip_error; 2026 rth->dst.error= -err; 2027 rth->rt_flags &= ~RTCF_LOCAL; 2028 } 2029 2030 if (do_cache) { 2031 struct fib_nh *nh = &FIB_RES_NH(*res); 2032 2033 rth->dst.lwtstate = lwtstate_get(nh->nh_lwtstate); 2034 if (lwtunnel_input_redirect(rth->dst.lwtstate)) { 2035 WARN_ON(rth->dst.input == lwtunnel_input); 2036 rth->dst.lwtstate->orig_input = rth->dst.input; 2037 rth->dst.input = lwtunnel_input; 2038 } 2039 2040 if (unlikely(!rt_cache_route(nh, rth))) 2041 rt_add_uncached_list(rth); 2042 } 2043 skb_dst_set(skb, &rth->dst); 2044 err = 0; 2045 goto out; 2046 2047 no_route: 2048 RT_CACHE_STAT_INC(in_no_route); 2049 res->type = RTN_UNREACHABLE; 2050 res->fi = NULL; 2051 res->table = NULL; 2052 goto local_input; 2053 2054 /* 2055 * Do not cache martian addresses: they should be logged (RFC1812) 2056 */ 2057 martian_destination: 2058 RT_CACHE_STAT_INC(in_martian_dst); 2059 #ifdef CONFIG_IP_ROUTE_VERBOSE 2060 if (IN_DEV_LOG_MARTIANS(in_dev)) 2061 net_warn_ratelimited("martian destination %pI4 from %pI4, dev %s\n", 2062 &daddr, &saddr, dev->name); 2063 #endif 2064 2065 e_inval: 2066 err = -EINVAL; 2067 goto out; 2068 2069 e_nobufs: 2070 err = -ENOBUFS; 2071 goto out; 2072 2073 martian_source: 2074 ip_handle_martian_source(dev, in_dev, skb, daddr, saddr); 2075 goto out; 2076 } 2077 2078 int ip_route_input_noref(struct sk_buff *skb, __be32 daddr, __be32 saddr, 2079 u8 tos, struct net_device *dev) 2080 { 2081 struct fib_result res; 2082 int err; 2083 2084 tos &= IPTOS_RT_MASK; 2085 rcu_read_lock(); 2086 err = ip_route_input_rcu(skb, daddr, saddr, tos, dev, &res); 2087 rcu_read_unlock(); 2088 2089 return err; 2090 } 2091 EXPORT_SYMBOL(ip_route_input_noref); 2092 2093 /* called with rcu_read_lock held */ 2094 int ip_route_input_rcu(struct sk_buff *skb, __be32 daddr, __be32 saddr, 2095 u8 tos, struct net_device *dev, struct fib_result *res) 2096 { 2097 /* Multicast recognition logic is moved from route cache to here. 2098 The problem was that too many Ethernet cards have broken/missing 2099 hardware multicast filters :-( As result the host on multicasting 2100 network acquires a lot of useless route cache entries, sort of 2101 SDR messages from all the world. Now we try to get rid of them. 2102 Really, provided software IP multicast filter is organized 2103 reasonably (at least, hashed), it does not result in a slowdown 2104 comparing with route cache reject entries. 2105 Note, that multicast routers are not affected, because 2106 route cache entry is created eventually. 2107 */ 2108 if (ipv4_is_multicast(daddr)) { 2109 struct in_device *in_dev = __in_dev_get_rcu(dev); 2110 int our = 0; 2111 int err = -EINVAL; 2112 2113 if (in_dev) 2114 our = ip_check_mc_rcu(in_dev, daddr, saddr, 2115 ip_hdr(skb)->protocol); 2116 2117 /* check l3 master if no match yet */ 2118 if ((!in_dev || !our) && netif_is_l3_slave(dev)) { 2119 struct in_device *l3_in_dev; 2120 2121 l3_in_dev = __in_dev_get_rcu(skb->dev); 2122 if (l3_in_dev) 2123 our = ip_check_mc_rcu(l3_in_dev, daddr, saddr, 2124 ip_hdr(skb)->protocol); 2125 } 2126 2127 if (our 2128 #ifdef CONFIG_IP_MROUTE 2129 || 2130 (!ipv4_is_local_multicast(daddr) && 2131 IN_DEV_MFORWARD(in_dev)) 2132 #endif 2133 ) { 2134 err = ip_route_input_mc(skb, daddr, saddr, 2135 tos, dev, our); 2136 } 2137 return err; 2138 } 2139 2140 return ip_route_input_slow(skb, daddr, saddr, tos, dev, res); 2141 } 2142 2143 /* called with rcu_read_lock() */ 2144 static struct rtable *__mkroute_output(const struct fib_result *res, 2145 const struct flowi4 *fl4, int orig_oif, 2146 struct net_device *dev_out, 2147 unsigned int flags) 2148 { 2149 struct fib_info *fi = res->fi; 2150 struct fib_nh_exception *fnhe; 2151 struct in_device *in_dev; 2152 u16 type = res->type; 2153 struct rtable *rth; 2154 bool do_cache; 2155 2156 in_dev = __in_dev_get_rcu(dev_out); 2157 if (!in_dev) 2158 return ERR_PTR(-EINVAL); 2159 2160 if (likely(!IN_DEV_ROUTE_LOCALNET(in_dev))) 2161 if (ipv4_is_loopback(fl4->saddr) && 2162 !(dev_out->flags & IFF_LOOPBACK) && 2163 !netif_is_l3_master(dev_out)) 2164 return ERR_PTR(-EINVAL); 2165 2166 if (ipv4_is_lbcast(fl4->daddr)) 2167 type = RTN_BROADCAST; 2168 else if (ipv4_is_multicast(fl4->daddr)) 2169 type = RTN_MULTICAST; 2170 else if (ipv4_is_zeronet(fl4->daddr)) 2171 return ERR_PTR(-EINVAL); 2172 2173 if (dev_out->flags & IFF_LOOPBACK) 2174 flags |= RTCF_LOCAL; 2175 2176 do_cache = true; 2177 if (type == RTN_BROADCAST) { 2178 flags |= RTCF_BROADCAST | RTCF_LOCAL; 2179 fi = NULL; 2180 } else if (type == RTN_MULTICAST) { 2181 flags |= RTCF_MULTICAST | RTCF_LOCAL; 2182 if (!ip_check_mc_rcu(in_dev, fl4->daddr, fl4->saddr, 2183 fl4->flowi4_proto)) 2184 flags &= ~RTCF_LOCAL; 2185 else 2186 do_cache = false; 2187 /* If multicast route do not exist use 2188 * default one, but do not gateway in this case. 2189 * Yes, it is hack. 2190 */ 2191 if (fi && res->prefixlen < 4) 2192 fi = NULL; 2193 } else if ((type == RTN_LOCAL) && (orig_oif != 0) && 2194 (orig_oif != dev_out->ifindex)) { 2195 /* For local routes that require a particular output interface 2196 * we do not want to cache the result. Caching the result 2197 * causes incorrect behaviour when there are multiple source 2198 * addresses on the interface, the end result being that if the 2199 * intended recipient is waiting on that interface for the 2200 * packet he won't receive it because it will be delivered on 2201 * the loopback interface and the IP_PKTINFO ipi_ifindex will 2202 * be set to the loopback interface as well. 2203 */ 2204 fi = NULL; 2205 } 2206 2207 fnhe = NULL; 2208 do_cache &= fi != NULL; 2209 if (do_cache) { 2210 struct rtable __rcu **prth; 2211 struct fib_nh *nh = &FIB_RES_NH(*res); 2212 2213 fnhe = find_exception(nh, fl4->daddr); 2214 if (fnhe) { 2215 prth = &fnhe->fnhe_rth_output; 2216 rth = rcu_dereference(*prth); 2217 if (rth && rth->dst.expires && 2218 time_after(jiffies, rth->dst.expires)) { 2219 ip_del_fnhe(nh, fl4->daddr); 2220 fnhe = NULL; 2221 } else { 2222 goto rt_cache; 2223 } 2224 } 2225 2226 if (unlikely(fl4->flowi4_flags & 2227 FLOWI_FLAG_KNOWN_NH && 2228 !(nh->nh_gw && 2229 nh->nh_scope == RT_SCOPE_LINK))) { 2230 do_cache = false; 2231 goto add; 2232 } 2233 prth = raw_cpu_ptr(nh->nh_pcpu_rth_output); 2234 rth = rcu_dereference(*prth); 2235 2236 rt_cache: 2237 if (rt_cache_valid(rth) && dst_hold_safe(&rth->dst)) 2238 return rth; 2239 } 2240 2241 add: 2242 rth = rt_dst_alloc(dev_out, flags, type, 2243 IN_DEV_CONF_GET(in_dev, NOPOLICY), 2244 IN_DEV_CONF_GET(in_dev, NOXFRM), 2245 do_cache); 2246 if (!rth) 2247 return ERR_PTR(-ENOBUFS); 2248 2249 rth->rt_iif = orig_oif; 2250 if (res->table) 2251 rth->rt_table_id = res->table->tb_id; 2252 2253 RT_CACHE_STAT_INC(out_slow_tot); 2254 2255 if (flags & (RTCF_BROADCAST | RTCF_MULTICAST)) { 2256 if (flags & RTCF_LOCAL && 2257 !(dev_out->flags & IFF_LOOPBACK)) { 2258 rth->dst.output = ip_mc_output; 2259 RT_CACHE_STAT_INC(out_slow_mc); 2260 } 2261 #ifdef CONFIG_IP_MROUTE 2262 if (type == RTN_MULTICAST) { 2263 if (IN_DEV_MFORWARD(in_dev) && 2264 !ipv4_is_local_multicast(fl4->daddr)) { 2265 rth->dst.input = ip_mr_input; 2266 rth->dst.output = ip_mc_output; 2267 } 2268 } 2269 #endif 2270 } 2271 2272 rt_set_nexthop(rth, fl4->daddr, res, fnhe, fi, type, 0, do_cache); 2273 set_lwt_redirect(rth); 2274 2275 return rth; 2276 } 2277 2278 /* 2279 * Major route resolver routine. 2280 */ 2281 2282 struct rtable *ip_route_output_key_hash(struct net *net, struct flowi4 *fl4, 2283 const struct sk_buff *skb) 2284 { 2285 __u8 tos = RT_FL_TOS(fl4); 2286 struct fib_result res; 2287 struct rtable *rth; 2288 2289 res.tclassid = 0; 2290 res.fi = NULL; 2291 res.table = NULL; 2292 2293 fl4->flowi4_iif = LOOPBACK_IFINDEX; 2294 fl4->flowi4_tos = tos & IPTOS_RT_MASK; 2295 fl4->flowi4_scope = ((tos & RTO_ONLINK) ? 2296 RT_SCOPE_LINK : RT_SCOPE_UNIVERSE); 2297 2298 rcu_read_lock(); 2299 rth = ip_route_output_key_hash_rcu(net, fl4, &res, skb); 2300 rcu_read_unlock(); 2301 2302 return rth; 2303 } 2304 EXPORT_SYMBOL_GPL(ip_route_output_key_hash); 2305 2306 struct rtable *ip_route_output_key_hash_rcu(struct net *net, struct flowi4 *fl4, 2307 struct fib_result *res, 2308 const struct sk_buff *skb) 2309 { 2310 struct net_device *dev_out = NULL; 2311 int orig_oif = fl4->flowi4_oif; 2312 unsigned int flags = 0; 2313 struct rtable *rth; 2314 int err = -ENETUNREACH; 2315 2316 if (fl4->saddr) { 2317 rth = ERR_PTR(-EINVAL); 2318 if (ipv4_is_multicast(fl4->saddr) || 2319 ipv4_is_lbcast(fl4->saddr) || 2320 ipv4_is_zeronet(fl4->saddr)) 2321 goto out; 2322 2323 /* I removed check for oif == dev_out->oif here. 2324 It was wrong for two reasons: 2325 1. ip_dev_find(net, saddr) can return wrong iface, if saddr 2326 is assigned to multiple interfaces. 2327 2. Moreover, we are allowed to send packets with saddr 2328 of another iface. --ANK 2329 */ 2330 2331 if (fl4->flowi4_oif == 0 && 2332 (ipv4_is_multicast(fl4->daddr) || 2333 ipv4_is_lbcast(fl4->daddr))) { 2334 /* It is equivalent to inet_addr_type(saddr) == RTN_LOCAL */ 2335 dev_out = __ip_dev_find(net, fl4->saddr, false); 2336 if (!dev_out) 2337 goto out; 2338 2339 /* Special hack: user can direct multicasts 2340 and limited broadcast via necessary interface 2341 without fiddling with IP_MULTICAST_IF or IP_PKTINFO. 2342 This hack is not just for fun, it allows 2343 vic,vat and friends to work. 2344 They bind socket to loopback, set ttl to zero 2345 and expect that it will work. 2346 From the viewpoint of routing cache they are broken, 2347 because we are not allowed to build multicast path 2348 with loopback source addr (look, routing cache 2349 cannot know, that ttl is zero, so that packet 2350 will not leave this host and route is valid). 2351 Luckily, this hack is good workaround. 2352 */ 2353 2354 fl4->flowi4_oif = dev_out->ifindex; 2355 goto make_route; 2356 } 2357 2358 if (!(fl4->flowi4_flags & FLOWI_FLAG_ANYSRC)) { 2359 /* It is equivalent to inet_addr_type(saddr) == RTN_LOCAL */ 2360 if (!__ip_dev_find(net, fl4->saddr, false)) 2361 goto out; 2362 } 2363 } 2364 2365 2366 if (fl4->flowi4_oif) { 2367 dev_out = dev_get_by_index_rcu(net, fl4->flowi4_oif); 2368 rth = ERR_PTR(-ENODEV); 2369 if (!dev_out) 2370 goto out; 2371 2372 /* RACE: Check return value of inet_select_addr instead. */ 2373 if (!(dev_out->flags & IFF_UP) || !__in_dev_get_rcu(dev_out)) { 2374 rth = ERR_PTR(-ENETUNREACH); 2375 goto out; 2376 } 2377 if (ipv4_is_local_multicast(fl4->daddr) || 2378 ipv4_is_lbcast(fl4->daddr) || 2379 fl4->flowi4_proto == IPPROTO_IGMP) { 2380 if (!fl4->saddr) 2381 fl4->saddr = inet_select_addr(dev_out, 0, 2382 RT_SCOPE_LINK); 2383 goto make_route; 2384 } 2385 if (!fl4->saddr) { 2386 if (ipv4_is_multicast(fl4->daddr)) 2387 fl4->saddr = inet_select_addr(dev_out, 0, 2388 fl4->flowi4_scope); 2389 else if (!fl4->daddr) 2390 fl4->saddr = inet_select_addr(dev_out, 0, 2391 RT_SCOPE_HOST); 2392 } 2393 } 2394 2395 if (!fl4->daddr) { 2396 fl4->daddr = fl4->saddr; 2397 if (!fl4->daddr) 2398 fl4->daddr = fl4->saddr = htonl(INADDR_LOOPBACK); 2399 dev_out = net->loopback_dev; 2400 fl4->flowi4_oif = LOOPBACK_IFINDEX; 2401 res->type = RTN_LOCAL; 2402 flags |= RTCF_LOCAL; 2403 goto make_route; 2404 } 2405 2406 err = fib_lookup(net, fl4, res, 0); 2407 if (err) { 2408 res->fi = NULL; 2409 res->table = NULL; 2410 if (fl4->flowi4_oif && 2411 (ipv4_is_multicast(fl4->daddr) || 2412 !netif_index_is_l3_master(net, fl4->flowi4_oif))) { 2413 /* Apparently, routing tables are wrong. Assume, 2414 that the destination is on link. 2415 2416 WHY? DW. 2417 Because we are allowed to send to iface 2418 even if it has NO routes and NO assigned 2419 addresses. When oif is specified, routing 2420 tables are looked up with only one purpose: 2421 to catch if destination is gatewayed, rather than 2422 direct. Moreover, if MSG_DONTROUTE is set, 2423 we send packet, ignoring both routing tables 2424 and ifaddr state. --ANK 2425 2426 2427 We could make it even if oif is unknown, 2428 likely IPv6, but we do not. 2429 */ 2430 2431 if (fl4->saddr == 0) 2432 fl4->saddr = inet_select_addr(dev_out, 0, 2433 RT_SCOPE_LINK); 2434 res->type = RTN_UNICAST; 2435 goto make_route; 2436 } 2437 rth = ERR_PTR(err); 2438 goto out; 2439 } 2440 2441 if (res->type == RTN_LOCAL) { 2442 if (!fl4->saddr) { 2443 if (res->fi->fib_prefsrc) 2444 fl4->saddr = res->fi->fib_prefsrc; 2445 else 2446 fl4->saddr = fl4->daddr; 2447 } 2448 2449 /* L3 master device is the loopback for that domain */ 2450 dev_out = l3mdev_master_dev_rcu(FIB_RES_DEV(*res)) ? : 2451 net->loopback_dev; 2452 2453 /* make sure orig_oif points to fib result device even 2454 * though packet rx/tx happens over loopback or l3mdev 2455 */ 2456 orig_oif = FIB_RES_OIF(*res); 2457 2458 fl4->flowi4_oif = dev_out->ifindex; 2459 flags |= RTCF_LOCAL; 2460 goto make_route; 2461 } 2462 2463 fib_select_path(net, res, fl4, skb); 2464 2465 dev_out = FIB_RES_DEV(*res); 2466 fl4->flowi4_oif = dev_out->ifindex; 2467 2468 2469 make_route: 2470 rth = __mkroute_output(res, fl4, orig_oif, dev_out, flags); 2471 2472 out: 2473 return rth; 2474 } 2475 2476 static struct dst_entry *ipv4_blackhole_dst_check(struct dst_entry *dst, u32 cookie) 2477 { 2478 return NULL; 2479 } 2480 2481 static unsigned int ipv4_blackhole_mtu(const struct dst_entry *dst) 2482 { 2483 unsigned int mtu = dst_metric_raw(dst, RTAX_MTU); 2484 2485 return mtu ? : dst->dev->mtu; 2486 } 2487 2488 static void ipv4_rt_blackhole_update_pmtu(struct dst_entry *dst, struct sock *sk, 2489 struct sk_buff *skb, u32 mtu) 2490 { 2491 } 2492 2493 static void ipv4_rt_blackhole_redirect(struct dst_entry *dst, struct sock *sk, 2494 struct sk_buff *skb) 2495 { 2496 } 2497 2498 static u32 *ipv4_rt_blackhole_cow_metrics(struct dst_entry *dst, 2499 unsigned long old) 2500 { 2501 return NULL; 2502 } 2503 2504 static struct dst_ops ipv4_dst_blackhole_ops = { 2505 .family = AF_INET, 2506 .check = ipv4_blackhole_dst_check, 2507 .mtu = ipv4_blackhole_mtu, 2508 .default_advmss = ipv4_default_advmss, 2509 .update_pmtu = ipv4_rt_blackhole_update_pmtu, 2510 .redirect = ipv4_rt_blackhole_redirect, 2511 .cow_metrics = ipv4_rt_blackhole_cow_metrics, 2512 .neigh_lookup = ipv4_neigh_lookup, 2513 }; 2514 2515 struct dst_entry *ipv4_blackhole_route(struct net *net, struct dst_entry *dst_orig) 2516 { 2517 struct rtable *ort = (struct rtable *) dst_orig; 2518 struct rtable *rt; 2519 2520 rt = dst_alloc(&ipv4_dst_blackhole_ops, NULL, 1, DST_OBSOLETE_DEAD, 0); 2521 if (rt) { 2522 struct dst_entry *new = &rt->dst; 2523 2524 new->__use = 1; 2525 new->input = dst_discard; 2526 new->output = dst_discard_out; 2527 2528 new->dev = net->loopback_dev; 2529 if (new->dev) 2530 dev_hold(new->dev); 2531 2532 rt->rt_is_input = ort->rt_is_input; 2533 rt->rt_iif = ort->rt_iif; 2534 rt->rt_pmtu = ort->rt_pmtu; 2535 2536 rt->rt_genid = rt_genid_ipv4(net); 2537 rt->rt_flags = ort->rt_flags; 2538 rt->rt_type = ort->rt_type; 2539 rt->rt_gateway = ort->rt_gateway; 2540 rt->rt_uses_gateway = ort->rt_uses_gateway; 2541 2542 INIT_LIST_HEAD(&rt->rt_uncached); 2543 } 2544 2545 dst_release(dst_orig); 2546 2547 return rt ? &rt->dst : ERR_PTR(-ENOMEM); 2548 } 2549 2550 struct rtable *ip_route_output_flow(struct net *net, struct flowi4 *flp4, 2551 const struct sock *sk) 2552 { 2553 struct rtable *rt = __ip_route_output_key(net, flp4); 2554 2555 if (IS_ERR(rt)) 2556 return rt; 2557 2558 if (flp4->flowi4_proto) 2559 rt = (struct rtable *)xfrm_lookup_route(net, &rt->dst, 2560 flowi4_to_flowi(flp4), 2561 sk, 0); 2562 2563 return rt; 2564 } 2565 EXPORT_SYMBOL_GPL(ip_route_output_flow); 2566 2567 /* called with rcu_read_lock held */ 2568 static int rt_fill_info(struct net *net, __be32 dst, __be32 src, u32 table_id, 2569 struct flowi4 *fl4, struct sk_buff *skb, u32 portid, 2570 u32 seq) 2571 { 2572 struct rtable *rt = skb_rtable(skb); 2573 struct rtmsg *r; 2574 struct nlmsghdr *nlh; 2575 unsigned long expires = 0; 2576 u32 error; 2577 u32 metrics[RTAX_MAX]; 2578 2579 nlh = nlmsg_put(skb, portid, seq, RTM_NEWROUTE, sizeof(*r), 0); 2580 if (!nlh) 2581 return -EMSGSIZE; 2582 2583 r = nlmsg_data(nlh); 2584 r->rtm_family = AF_INET; 2585 r->rtm_dst_len = 32; 2586 r->rtm_src_len = 0; 2587 r->rtm_tos = fl4->flowi4_tos; 2588 r->rtm_table = table_id < 256 ? table_id : RT_TABLE_COMPAT; 2589 if (nla_put_u32(skb, RTA_TABLE, table_id)) 2590 goto nla_put_failure; 2591 r->rtm_type = rt->rt_type; 2592 r->rtm_scope = RT_SCOPE_UNIVERSE; 2593 r->rtm_protocol = RTPROT_UNSPEC; 2594 r->rtm_flags = (rt->rt_flags & ~0xFFFF) | RTM_F_CLONED; 2595 if (rt->rt_flags & RTCF_NOTIFY) 2596 r->rtm_flags |= RTM_F_NOTIFY; 2597 if (IPCB(skb)->flags & IPSKB_DOREDIRECT) 2598 r->rtm_flags |= RTCF_DOREDIRECT; 2599 2600 if (nla_put_in_addr(skb, RTA_DST, dst)) 2601 goto nla_put_failure; 2602 if (src) { 2603 r->rtm_src_len = 32; 2604 if (nla_put_in_addr(skb, RTA_SRC, src)) 2605 goto nla_put_failure; 2606 } 2607 if (rt->dst.dev && 2608 nla_put_u32(skb, RTA_OIF, rt->dst.dev->ifindex)) 2609 goto nla_put_failure; 2610 #ifdef CONFIG_IP_ROUTE_CLASSID 2611 if (rt->dst.tclassid && 2612 nla_put_u32(skb, RTA_FLOW, rt->dst.tclassid)) 2613 goto nla_put_failure; 2614 #endif 2615 if (!rt_is_input_route(rt) && 2616 fl4->saddr != src) { 2617 if (nla_put_in_addr(skb, RTA_PREFSRC, fl4->saddr)) 2618 goto nla_put_failure; 2619 } 2620 if (rt->rt_uses_gateway && 2621 nla_put_in_addr(skb, RTA_GATEWAY, rt->rt_gateway)) 2622 goto nla_put_failure; 2623 2624 expires = rt->dst.expires; 2625 if (expires) { 2626 unsigned long now = jiffies; 2627 2628 if (time_before(now, expires)) 2629 expires -= now; 2630 else 2631 expires = 0; 2632 } 2633 2634 memcpy(metrics, dst_metrics_ptr(&rt->dst), sizeof(metrics)); 2635 if (rt->rt_pmtu && expires) 2636 metrics[RTAX_MTU - 1] = rt->rt_pmtu; 2637 if (rtnetlink_put_metrics(skb, metrics) < 0) 2638 goto nla_put_failure; 2639 2640 if (fl4->flowi4_mark && 2641 nla_put_u32(skb, RTA_MARK, fl4->flowi4_mark)) 2642 goto nla_put_failure; 2643 2644 if (!uid_eq(fl4->flowi4_uid, INVALID_UID) && 2645 nla_put_u32(skb, RTA_UID, 2646 from_kuid_munged(current_user_ns(), fl4->flowi4_uid))) 2647 goto nla_put_failure; 2648 2649 error = rt->dst.error; 2650 2651 if (rt_is_input_route(rt)) { 2652 #ifdef CONFIG_IP_MROUTE 2653 if (ipv4_is_multicast(dst) && !ipv4_is_local_multicast(dst) && 2654 IPV4_DEVCONF_ALL(net, MC_FORWARDING)) { 2655 int err = ipmr_get_route(net, skb, 2656 fl4->saddr, fl4->daddr, 2657 r, portid); 2658 2659 if (err <= 0) { 2660 if (err == 0) 2661 return 0; 2662 goto nla_put_failure; 2663 } 2664 } else 2665 #endif 2666 if (nla_put_u32(skb, RTA_IIF, skb->dev->ifindex)) 2667 goto nla_put_failure; 2668 } 2669 2670 if (rtnl_put_cacheinfo(skb, &rt->dst, 0, expires, error) < 0) 2671 goto nla_put_failure; 2672 2673 nlmsg_end(skb, nlh); 2674 return 0; 2675 2676 nla_put_failure: 2677 nlmsg_cancel(skb, nlh); 2678 return -EMSGSIZE; 2679 } 2680 2681 static int inet_rtm_getroute(struct sk_buff *in_skb, struct nlmsghdr *nlh, 2682 struct netlink_ext_ack *extack) 2683 { 2684 struct net *net = sock_net(in_skb->sk); 2685 struct rtmsg *rtm; 2686 struct nlattr *tb[RTA_MAX+1]; 2687 struct fib_result res = {}; 2688 struct rtable *rt = NULL; 2689 struct flowi4 fl4; 2690 __be32 dst = 0; 2691 __be32 src = 0; 2692 u32 iif; 2693 int err; 2694 int mark; 2695 struct sk_buff *skb; 2696 u32 table_id = RT_TABLE_MAIN; 2697 kuid_t uid; 2698 2699 err = nlmsg_parse(nlh, sizeof(*rtm), tb, RTA_MAX, rtm_ipv4_policy, 2700 extack); 2701 if (err < 0) 2702 goto errout; 2703 2704 rtm = nlmsg_data(nlh); 2705 2706 skb = alloc_skb(NLMSG_GOODSIZE, GFP_KERNEL); 2707 if (!skb) { 2708 err = -ENOBUFS; 2709 goto errout; 2710 } 2711 2712 /* Reserve room for dummy headers, this skb can pass 2713 through good chunk of routing engine. 2714 */ 2715 skb_reset_mac_header(skb); 2716 skb_reset_network_header(skb); 2717 2718 src = tb[RTA_SRC] ? nla_get_in_addr(tb[RTA_SRC]) : 0; 2719 dst = tb[RTA_DST] ? nla_get_in_addr(tb[RTA_DST]) : 0; 2720 iif = tb[RTA_IIF] ? nla_get_u32(tb[RTA_IIF]) : 0; 2721 mark = tb[RTA_MARK] ? nla_get_u32(tb[RTA_MARK]) : 0; 2722 if (tb[RTA_UID]) 2723 uid = make_kuid(current_user_ns(), nla_get_u32(tb[RTA_UID])); 2724 else 2725 uid = (iif ? INVALID_UID : current_uid()); 2726 2727 /* Bugfix: need to give ip_route_input enough of an IP header to 2728 * not gag. 2729 */ 2730 ip_hdr(skb)->protocol = IPPROTO_UDP; 2731 ip_hdr(skb)->saddr = src; 2732 ip_hdr(skb)->daddr = dst; 2733 2734 skb_reserve(skb, MAX_HEADER + sizeof(struct iphdr)); 2735 2736 memset(&fl4, 0, sizeof(fl4)); 2737 fl4.daddr = dst; 2738 fl4.saddr = src; 2739 fl4.flowi4_tos = rtm->rtm_tos; 2740 fl4.flowi4_oif = tb[RTA_OIF] ? nla_get_u32(tb[RTA_OIF]) : 0; 2741 fl4.flowi4_mark = mark; 2742 fl4.flowi4_uid = uid; 2743 2744 rcu_read_lock(); 2745 2746 if (iif) { 2747 struct net_device *dev; 2748 2749 dev = dev_get_by_index_rcu(net, iif); 2750 if (!dev) { 2751 err = -ENODEV; 2752 goto errout_free; 2753 } 2754 2755 skb->protocol = htons(ETH_P_IP); 2756 skb->dev = dev; 2757 skb->mark = mark; 2758 err = ip_route_input_rcu(skb, dst, src, rtm->rtm_tos, 2759 dev, &res); 2760 2761 rt = skb_rtable(skb); 2762 if (err == 0 && rt->dst.error) 2763 err = -rt->dst.error; 2764 } else { 2765 rt = ip_route_output_key_hash_rcu(net, &fl4, &res, skb); 2766 err = 0; 2767 if (IS_ERR(rt)) 2768 err = PTR_ERR(rt); 2769 else 2770 skb_dst_set(skb, &rt->dst); 2771 } 2772 2773 if (err) 2774 goto errout_free; 2775 2776 if (rtm->rtm_flags & RTM_F_NOTIFY) 2777 rt->rt_flags |= RTCF_NOTIFY; 2778 2779 if (rtm->rtm_flags & RTM_F_LOOKUP_TABLE) 2780 table_id = rt->rt_table_id; 2781 2782 if (rtm->rtm_flags & RTM_F_FIB_MATCH) { 2783 if (!res.fi) { 2784 err = fib_props[res.type].error; 2785 if (!err) 2786 err = -EHOSTUNREACH; 2787 goto errout_free; 2788 } 2789 err = fib_dump_info(skb, NETLINK_CB(in_skb).portid, 2790 nlh->nlmsg_seq, RTM_NEWROUTE, table_id, 2791 rt->rt_type, res.prefix, res.prefixlen, 2792 fl4.flowi4_tos, res.fi, 0); 2793 } else { 2794 err = rt_fill_info(net, dst, src, table_id, &fl4, skb, 2795 NETLINK_CB(in_skb).portid, nlh->nlmsg_seq); 2796 } 2797 if (err < 0) 2798 goto errout_free; 2799 2800 rcu_read_unlock(); 2801 2802 err = rtnl_unicast(skb, net, NETLINK_CB(in_skb).portid); 2803 errout: 2804 return err; 2805 2806 errout_free: 2807 rcu_read_unlock(); 2808 kfree_skb(skb); 2809 goto errout; 2810 } 2811 2812 void ip_rt_multicast_event(struct in_device *in_dev) 2813 { 2814 rt_cache_flush(dev_net(in_dev->dev)); 2815 } 2816 2817 #ifdef CONFIG_SYSCTL 2818 static int ip_rt_gc_interval __read_mostly = 60 * HZ; 2819 static int ip_rt_gc_min_interval __read_mostly = HZ / 2; 2820 static int ip_rt_gc_elasticity __read_mostly = 8; 2821 2822 static int ipv4_sysctl_rtcache_flush(struct ctl_table *__ctl, int write, 2823 void __user *buffer, 2824 size_t *lenp, loff_t *ppos) 2825 { 2826 struct net *net = (struct net *)__ctl->extra1; 2827 2828 if (write) { 2829 rt_cache_flush(net); 2830 fnhe_genid_bump(net); 2831 return 0; 2832 } 2833 2834 return -EINVAL; 2835 } 2836 2837 static struct ctl_table ipv4_route_table[] = { 2838 { 2839 .procname = "gc_thresh", 2840 .data = &ipv4_dst_ops.gc_thresh, 2841 .maxlen = sizeof(int), 2842 .mode = 0644, 2843 .proc_handler = proc_dointvec, 2844 }, 2845 { 2846 .procname = "max_size", 2847 .data = &ip_rt_max_size, 2848 .maxlen = sizeof(int), 2849 .mode = 0644, 2850 .proc_handler = proc_dointvec, 2851 }, 2852 { 2853 /* Deprecated. Use gc_min_interval_ms */ 2854 2855 .procname = "gc_min_interval", 2856 .data = &ip_rt_gc_min_interval, 2857 .maxlen = sizeof(int), 2858 .mode = 0644, 2859 .proc_handler = proc_dointvec_jiffies, 2860 }, 2861 { 2862 .procname = "gc_min_interval_ms", 2863 .data = &ip_rt_gc_min_interval, 2864 .maxlen = sizeof(int), 2865 .mode = 0644, 2866 .proc_handler = proc_dointvec_ms_jiffies, 2867 }, 2868 { 2869 .procname = "gc_timeout", 2870 .data = &ip_rt_gc_timeout, 2871 .maxlen = sizeof(int), 2872 .mode = 0644, 2873 .proc_handler = proc_dointvec_jiffies, 2874 }, 2875 { 2876 .procname = "gc_interval", 2877 .data = &ip_rt_gc_interval, 2878 .maxlen = sizeof(int), 2879 .mode = 0644, 2880 .proc_handler = proc_dointvec_jiffies, 2881 }, 2882 { 2883 .procname = "redirect_load", 2884 .data = &ip_rt_redirect_load, 2885 .maxlen = sizeof(int), 2886 .mode = 0644, 2887 .proc_handler = proc_dointvec, 2888 }, 2889 { 2890 .procname = "redirect_number", 2891 .data = &ip_rt_redirect_number, 2892 .maxlen = sizeof(int), 2893 .mode = 0644, 2894 .proc_handler = proc_dointvec, 2895 }, 2896 { 2897 .procname = "redirect_silence", 2898 .data = &ip_rt_redirect_silence, 2899 .maxlen = sizeof(int), 2900 .mode = 0644, 2901 .proc_handler = proc_dointvec, 2902 }, 2903 { 2904 .procname = "error_cost", 2905 .data = &ip_rt_error_cost, 2906 .maxlen = sizeof(int), 2907 .mode = 0644, 2908 .proc_handler = proc_dointvec, 2909 }, 2910 { 2911 .procname = "error_burst", 2912 .data = &ip_rt_error_burst, 2913 .maxlen = sizeof(int), 2914 .mode = 0644, 2915 .proc_handler = proc_dointvec, 2916 }, 2917 { 2918 .procname = "gc_elasticity", 2919 .data = &ip_rt_gc_elasticity, 2920 .maxlen = sizeof(int), 2921 .mode = 0644, 2922 .proc_handler = proc_dointvec, 2923 }, 2924 { 2925 .procname = "mtu_expires", 2926 .data = &ip_rt_mtu_expires, 2927 .maxlen = sizeof(int), 2928 .mode = 0644, 2929 .proc_handler = proc_dointvec_jiffies, 2930 }, 2931 { 2932 .procname = "min_pmtu", 2933 .data = &ip_rt_min_pmtu, 2934 .maxlen = sizeof(int), 2935 .mode = 0644, 2936 .proc_handler = proc_dointvec, 2937 }, 2938 { 2939 .procname = "min_adv_mss", 2940 .data = &ip_rt_min_advmss, 2941 .maxlen = sizeof(int), 2942 .mode = 0644, 2943 .proc_handler = proc_dointvec, 2944 }, 2945 { } 2946 }; 2947 2948 static struct ctl_table ipv4_route_flush_table[] = { 2949 { 2950 .procname = "flush", 2951 .maxlen = sizeof(int), 2952 .mode = 0200, 2953 .proc_handler = ipv4_sysctl_rtcache_flush, 2954 }, 2955 { }, 2956 }; 2957 2958 static __net_init int sysctl_route_net_init(struct net *net) 2959 { 2960 struct ctl_table *tbl; 2961 2962 tbl = ipv4_route_flush_table; 2963 if (!net_eq(net, &init_net)) { 2964 tbl = kmemdup(tbl, sizeof(ipv4_route_flush_table), GFP_KERNEL); 2965 if (!tbl) 2966 goto err_dup; 2967 2968 /* Don't export sysctls to unprivileged users */ 2969 if (net->user_ns != &init_user_ns) 2970 tbl[0].procname = NULL; 2971 } 2972 tbl[0].extra1 = net; 2973 2974 net->ipv4.route_hdr = register_net_sysctl(net, "net/ipv4/route", tbl); 2975 if (!net->ipv4.route_hdr) 2976 goto err_reg; 2977 return 0; 2978 2979 err_reg: 2980 if (tbl != ipv4_route_flush_table) 2981 kfree(tbl); 2982 err_dup: 2983 return -ENOMEM; 2984 } 2985 2986 static __net_exit void sysctl_route_net_exit(struct net *net) 2987 { 2988 struct ctl_table *tbl; 2989 2990 tbl = net->ipv4.route_hdr->ctl_table_arg; 2991 unregister_net_sysctl_table(net->ipv4.route_hdr); 2992 BUG_ON(tbl == ipv4_route_flush_table); 2993 kfree(tbl); 2994 } 2995 2996 static __net_initdata struct pernet_operations sysctl_route_ops = { 2997 .init = sysctl_route_net_init, 2998 .exit = sysctl_route_net_exit, 2999 }; 3000 #endif 3001 3002 static __net_init int rt_genid_init(struct net *net) 3003 { 3004 atomic_set(&net->ipv4.rt_genid, 0); 3005 atomic_set(&net->fnhe_genid, 0); 3006 atomic_set(&net->ipv4.dev_addr_genid, get_random_int()); 3007 return 0; 3008 } 3009 3010 static __net_initdata struct pernet_operations rt_genid_ops = { 3011 .init = rt_genid_init, 3012 }; 3013 3014 static int __net_init ipv4_inetpeer_init(struct net *net) 3015 { 3016 struct inet_peer_base *bp = kmalloc(sizeof(*bp), GFP_KERNEL); 3017 3018 if (!bp) 3019 return -ENOMEM; 3020 inet_peer_base_init(bp); 3021 net->ipv4.peers = bp; 3022 return 0; 3023 } 3024 3025 static void __net_exit ipv4_inetpeer_exit(struct net *net) 3026 { 3027 struct inet_peer_base *bp = net->ipv4.peers; 3028 3029 net->ipv4.peers = NULL; 3030 inetpeer_invalidate_tree(bp); 3031 kfree(bp); 3032 } 3033 3034 static __net_initdata struct pernet_operations ipv4_inetpeer_ops = { 3035 .init = ipv4_inetpeer_init, 3036 .exit = ipv4_inetpeer_exit, 3037 }; 3038 3039 #ifdef CONFIG_IP_ROUTE_CLASSID 3040 struct ip_rt_acct __percpu *ip_rt_acct __read_mostly; 3041 #endif /* CONFIG_IP_ROUTE_CLASSID */ 3042 3043 int __init ip_rt_init(void) 3044 { 3045 int cpu; 3046 3047 ip_idents = kmalloc(IP_IDENTS_SZ * sizeof(*ip_idents), GFP_KERNEL); 3048 if (!ip_idents) 3049 panic("IP: failed to allocate ip_idents\n"); 3050 3051 prandom_bytes(ip_idents, IP_IDENTS_SZ * sizeof(*ip_idents)); 3052 3053 ip_tstamps = kcalloc(IP_IDENTS_SZ, sizeof(*ip_tstamps), GFP_KERNEL); 3054 if (!ip_tstamps) 3055 panic("IP: failed to allocate ip_tstamps\n"); 3056 3057 for_each_possible_cpu(cpu) { 3058 struct uncached_list *ul = &per_cpu(rt_uncached_list, cpu); 3059 3060 INIT_LIST_HEAD(&ul->head); 3061 spin_lock_init(&ul->lock); 3062 } 3063 #ifdef CONFIG_IP_ROUTE_CLASSID 3064 ip_rt_acct = __alloc_percpu(256 * sizeof(struct ip_rt_acct), __alignof__(struct ip_rt_acct)); 3065 if (!ip_rt_acct) 3066 panic("IP: failed to allocate ip_rt_acct\n"); 3067 #endif 3068 3069 ipv4_dst_ops.kmem_cachep = 3070 kmem_cache_create("ip_dst_cache", sizeof(struct rtable), 0, 3071 SLAB_HWCACHE_ALIGN|SLAB_PANIC, NULL); 3072 3073 ipv4_dst_blackhole_ops.kmem_cachep = ipv4_dst_ops.kmem_cachep; 3074 3075 if (dst_entries_init(&ipv4_dst_ops) < 0) 3076 panic("IP: failed to allocate ipv4_dst_ops counter\n"); 3077 3078 if (dst_entries_init(&ipv4_dst_blackhole_ops) < 0) 3079 panic("IP: failed to allocate ipv4_dst_blackhole_ops counter\n"); 3080 3081 ipv4_dst_ops.gc_thresh = ~0; 3082 ip_rt_max_size = INT_MAX; 3083 3084 devinet_init(); 3085 ip_fib_init(); 3086 3087 if (ip_rt_proc_init()) 3088 pr_err("Unable to create route proc files\n"); 3089 #ifdef CONFIG_XFRM 3090 xfrm_init(); 3091 xfrm4_init(); 3092 #endif 3093 rtnl_register(PF_INET, RTM_GETROUTE, inet_rtm_getroute, NULL, 3094 RTNL_FLAG_DOIT_UNLOCKED); 3095 3096 #ifdef CONFIG_SYSCTL 3097 register_pernet_subsys(&sysctl_route_ops); 3098 #endif 3099 register_pernet_subsys(&rt_genid_ops); 3100 register_pernet_subsys(&ipv4_inetpeer_ops); 3101 return 0; 3102 } 3103 3104 #ifdef CONFIG_SYSCTL 3105 /* 3106 * We really need to sanitize the damn ipv4 init order, then all 3107 * this nonsense will go away. 3108 */ 3109 void __init ip_static_sysctl_init(void) 3110 { 3111 register_net_sysctl(&init_net, "net/ipv4/route", ipv4_route_table); 3112 } 3113 #endif 3114