1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * INET An implementation of the TCP/IP protocol suite for the LINUX 4 * operating system. INET is implemented using the BSD Socket 5 * interface as the means of communication with the user level. 6 * 7 * ROUTE - implementation of the IP router. 8 * 9 * Authors: Ross Biro 10 * Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG> 11 * Alan Cox, <gw4pts@gw4pts.ampr.org> 12 * Linus Torvalds, <Linus.Torvalds@helsinki.fi> 13 * Alexey Kuznetsov, <kuznet@ms2.inr.ac.ru> 14 * 15 * Fixes: 16 * Alan Cox : Verify area fixes. 17 * Alan Cox : cli() protects routing changes 18 * Rui Oliveira : ICMP routing table updates 19 * (rco@di.uminho.pt) Routing table insertion and update 20 * Linus Torvalds : Rewrote bits to be sensible 21 * Alan Cox : Added BSD route gw semantics 22 * Alan Cox : Super /proc >4K 23 * Alan Cox : MTU in route table 24 * Alan Cox : MSS actually. Also added the window 25 * clamper. 26 * Sam Lantinga : Fixed route matching in rt_del() 27 * Alan Cox : Routing cache support. 28 * Alan Cox : Removed compatibility cruft. 29 * Alan Cox : RTF_REJECT support. 30 * Alan Cox : TCP irtt support. 31 * Jonathan Naylor : Added Metric support. 32 * Miquel van Smoorenburg : BSD API fixes. 33 * Miquel van Smoorenburg : Metrics. 34 * Alan Cox : Use __u32 properly 35 * Alan Cox : Aligned routing errors more closely with BSD 36 * our system is still very different. 37 * Alan Cox : Faster /proc handling 38 * Alexey Kuznetsov : Massive rework to support tree based routing, 39 * routing caches and better behaviour. 40 * 41 * Olaf Erb : irtt wasn't being copied right. 42 * Bjorn Ekwall : Kerneld route support. 43 * Alan Cox : Multicast fixed (I hope) 44 * Pavel Krauz : Limited broadcast fixed 45 * Mike McLagan : Routing by source 46 * Alexey Kuznetsov : End of old history. Split to fib.c and 47 * route.c and rewritten from scratch. 48 * Andi Kleen : Load-limit warning messages. 49 * Vitaly E. Lavrov : Transparent proxy revived after year coma. 50 * Vitaly E. Lavrov : Race condition in ip_route_input_slow. 51 * Tobias Ringstrom : Uninitialized res.type in ip_route_output_slow. 52 * Vladimir V. Ivanov : IP rule info (flowid) is really useful. 53 * Marc Boucher : routing by fwmark 54 * Robert Olsson : Added rt_cache statistics 55 * Arnaldo C. Melo : Convert proc stuff to seq_file 56 * Eric Dumazet : hashed spinlocks and rt_check_expire() fixes. 57 * Ilia Sotnikov : Ignore TOS on PMTUD and Redirect 58 * Ilia Sotnikov : Removed TOS from hash calculations 59 */ 60 61 #define pr_fmt(fmt) "IPv4: " fmt 62 63 #include <linux/module.h> 64 #include <linux/uaccess.h> 65 #include <linux/bitops.h> 66 #include <linux/types.h> 67 #include <linux/kernel.h> 68 #include <linux/mm.h> 69 #include <linux/string.h> 70 #include <linux/socket.h> 71 #include <linux/sockios.h> 72 #include <linux/errno.h> 73 #include <linux/in.h> 74 #include <linux/inet.h> 75 #include <linux/netdevice.h> 76 #include <linux/proc_fs.h> 77 #include <linux/init.h> 78 #include <linux/skbuff.h> 79 #include <linux/inetdevice.h> 80 #include <linux/igmp.h> 81 #include <linux/pkt_sched.h> 82 #include <linux/mroute.h> 83 #include <linux/netfilter_ipv4.h> 84 #include <linux/random.h> 85 #include <linux/rcupdate.h> 86 #include <linux/times.h> 87 #include <linux/slab.h> 88 #include <linux/jhash.h> 89 #include <net/dst.h> 90 #include <net/dst_metadata.h> 91 #include <net/net_namespace.h> 92 #include <net/protocol.h> 93 #include <net/ip.h> 94 #include <net/route.h> 95 #include <net/inetpeer.h> 96 #include <net/sock.h> 97 #include <net/ip_fib.h> 98 #include <net/nexthop.h> 99 #include <net/arp.h> 100 #include <net/tcp.h> 101 #include <net/icmp.h> 102 #include <net/xfrm.h> 103 #include <net/lwtunnel.h> 104 #include <net/netevent.h> 105 #include <net/rtnetlink.h> 106 #ifdef CONFIG_SYSCTL 107 #include <linux/sysctl.h> 108 #endif 109 #include <net/secure_seq.h> 110 #include <net/ip_tunnels.h> 111 #include <net/l3mdev.h> 112 113 #include "fib_lookup.h" 114 115 #define RT_FL_TOS(oldflp4) \ 116 ((oldflp4)->flowi4_tos & (IPTOS_RT_MASK | RTO_ONLINK)) 117 118 #define RT_GC_TIMEOUT (300*HZ) 119 120 static int ip_rt_max_size; 121 static int ip_rt_redirect_number __read_mostly = 9; 122 static int ip_rt_redirect_load __read_mostly = HZ / 50; 123 static int ip_rt_redirect_silence __read_mostly = ((HZ / 50) << (9 + 1)); 124 static int ip_rt_error_cost __read_mostly = HZ; 125 static int ip_rt_error_burst __read_mostly = 5 * HZ; 126 static int ip_rt_mtu_expires __read_mostly = 10 * 60 * HZ; 127 static u32 ip_rt_min_pmtu __read_mostly = 512 + 20 + 20; 128 static int ip_rt_min_advmss __read_mostly = 256; 129 130 static int ip_rt_gc_timeout __read_mostly = RT_GC_TIMEOUT; 131 132 /* 133 * Interface to generic destination cache. 134 */ 135 136 INDIRECT_CALLABLE_SCOPE 137 struct dst_entry *ipv4_dst_check(struct dst_entry *dst, u32 cookie); 138 static unsigned int ipv4_default_advmss(const struct dst_entry *dst); 139 INDIRECT_CALLABLE_SCOPE 140 unsigned int ipv4_mtu(const struct dst_entry *dst); 141 static struct dst_entry *ipv4_negative_advice(struct dst_entry *dst); 142 static void ipv4_link_failure(struct sk_buff *skb); 143 static void ip_rt_update_pmtu(struct dst_entry *dst, struct sock *sk, 144 struct sk_buff *skb, u32 mtu, 145 bool confirm_neigh); 146 static void ip_do_redirect(struct dst_entry *dst, struct sock *sk, 147 struct sk_buff *skb); 148 static void ipv4_dst_destroy(struct dst_entry *dst); 149 150 static u32 *ipv4_cow_metrics(struct dst_entry *dst, unsigned long old) 151 { 152 WARN_ON(1); 153 return NULL; 154 } 155 156 static struct neighbour *ipv4_neigh_lookup(const struct dst_entry *dst, 157 struct sk_buff *skb, 158 const void *daddr); 159 static void ipv4_confirm_neigh(const struct dst_entry *dst, const void *daddr); 160 161 static struct dst_ops ipv4_dst_ops = { 162 .family = AF_INET, 163 .check = ipv4_dst_check, 164 .default_advmss = ipv4_default_advmss, 165 .mtu = ipv4_mtu, 166 .cow_metrics = ipv4_cow_metrics, 167 .destroy = ipv4_dst_destroy, 168 .negative_advice = ipv4_negative_advice, 169 .link_failure = ipv4_link_failure, 170 .update_pmtu = ip_rt_update_pmtu, 171 .redirect = ip_do_redirect, 172 .local_out = __ip_local_out, 173 .neigh_lookup = ipv4_neigh_lookup, 174 .confirm_neigh = ipv4_confirm_neigh, 175 }; 176 177 #define ECN_OR_COST(class) TC_PRIO_##class 178 179 const __u8 ip_tos2prio[16] = { 180 TC_PRIO_BESTEFFORT, 181 ECN_OR_COST(BESTEFFORT), 182 TC_PRIO_BESTEFFORT, 183 ECN_OR_COST(BESTEFFORT), 184 TC_PRIO_BULK, 185 ECN_OR_COST(BULK), 186 TC_PRIO_BULK, 187 ECN_OR_COST(BULK), 188 TC_PRIO_INTERACTIVE, 189 ECN_OR_COST(INTERACTIVE), 190 TC_PRIO_INTERACTIVE, 191 ECN_OR_COST(INTERACTIVE), 192 TC_PRIO_INTERACTIVE_BULK, 193 ECN_OR_COST(INTERACTIVE_BULK), 194 TC_PRIO_INTERACTIVE_BULK, 195 ECN_OR_COST(INTERACTIVE_BULK) 196 }; 197 EXPORT_SYMBOL(ip_tos2prio); 198 199 static DEFINE_PER_CPU(struct rt_cache_stat, rt_cache_stat); 200 #define RT_CACHE_STAT_INC(field) raw_cpu_inc(rt_cache_stat.field) 201 202 #ifdef CONFIG_PROC_FS 203 static void *rt_cache_seq_start(struct seq_file *seq, loff_t *pos) 204 { 205 if (*pos) 206 return NULL; 207 return SEQ_START_TOKEN; 208 } 209 210 static void *rt_cache_seq_next(struct seq_file *seq, void *v, loff_t *pos) 211 { 212 ++*pos; 213 return NULL; 214 } 215 216 static void rt_cache_seq_stop(struct seq_file *seq, void *v) 217 { 218 } 219 220 static int rt_cache_seq_show(struct seq_file *seq, void *v) 221 { 222 if (v == SEQ_START_TOKEN) 223 seq_printf(seq, "%-127s\n", 224 "Iface\tDestination\tGateway \tFlags\t\tRefCnt\tUse\t" 225 "Metric\tSource\t\tMTU\tWindow\tIRTT\tTOS\tHHRef\t" 226 "HHUptod\tSpecDst"); 227 return 0; 228 } 229 230 static const struct seq_operations rt_cache_seq_ops = { 231 .start = rt_cache_seq_start, 232 .next = rt_cache_seq_next, 233 .stop = rt_cache_seq_stop, 234 .show = rt_cache_seq_show, 235 }; 236 237 static int rt_cache_seq_open(struct inode *inode, struct file *file) 238 { 239 return seq_open(file, &rt_cache_seq_ops); 240 } 241 242 static const struct proc_ops rt_cache_proc_ops = { 243 .proc_open = rt_cache_seq_open, 244 .proc_read = seq_read, 245 .proc_lseek = seq_lseek, 246 .proc_release = seq_release, 247 }; 248 249 250 static void *rt_cpu_seq_start(struct seq_file *seq, loff_t *pos) 251 { 252 int cpu; 253 254 if (*pos == 0) 255 return SEQ_START_TOKEN; 256 257 for (cpu = *pos-1; cpu < nr_cpu_ids; ++cpu) { 258 if (!cpu_possible(cpu)) 259 continue; 260 *pos = cpu+1; 261 return &per_cpu(rt_cache_stat, cpu); 262 } 263 return NULL; 264 } 265 266 static void *rt_cpu_seq_next(struct seq_file *seq, void *v, loff_t *pos) 267 { 268 int cpu; 269 270 for (cpu = *pos; cpu < nr_cpu_ids; ++cpu) { 271 if (!cpu_possible(cpu)) 272 continue; 273 *pos = cpu+1; 274 return &per_cpu(rt_cache_stat, cpu); 275 } 276 (*pos)++; 277 return NULL; 278 279 } 280 281 static void rt_cpu_seq_stop(struct seq_file *seq, void *v) 282 { 283 284 } 285 286 static int rt_cpu_seq_show(struct seq_file *seq, void *v) 287 { 288 struct rt_cache_stat *st = v; 289 290 if (v == SEQ_START_TOKEN) { 291 seq_printf(seq, "entries in_hit in_slow_tot in_slow_mc in_no_route in_brd in_martian_dst in_martian_src out_hit out_slow_tot out_slow_mc gc_total gc_ignored gc_goal_miss gc_dst_overflow in_hlist_search out_hlist_search\n"); 292 return 0; 293 } 294 295 seq_printf(seq,"%08x %08x %08x %08x %08x %08x %08x %08x " 296 " %08x %08x %08x %08x %08x %08x %08x %08x %08x \n", 297 dst_entries_get_slow(&ipv4_dst_ops), 298 0, /* st->in_hit */ 299 st->in_slow_tot, 300 st->in_slow_mc, 301 st->in_no_route, 302 st->in_brd, 303 st->in_martian_dst, 304 st->in_martian_src, 305 306 0, /* st->out_hit */ 307 st->out_slow_tot, 308 st->out_slow_mc, 309 310 0, /* st->gc_total */ 311 0, /* st->gc_ignored */ 312 0, /* st->gc_goal_miss */ 313 0, /* st->gc_dst_overflow */ 314 0, /* st->in_hlist_search */ 315 0 /* st->out_hlist_search */ 316 ); 317 return 0; 318 } 319 320 static const struct seq_operations rt_cpu_seq_ops = { 321 .start = rt_cpu_seq_start, 322 .next = rt_cpu_seq_next, 323 .stop = rt_cpu_seq_stop, 324 .show = rt_cpu_seq_show, 325 }; 326 327 328 static int rt_cpu_seq_open(struct inode *inode, struct file *file) 329 { 330 return seq_open(file, &rt_cpu_seq_ops); 331 } 332 333 static const struct proc_ops rt_cpu_proc_ops = { 334 .proc_open = rt_cpu_seq_open, 335 .proc_read = seq_read, 336 .proc_lseek = seq_lseek, 337 .proc_release = seq_release, 338 }; 339 340 #ifdef CONFIG_IP_ROUTE_CLASSID 341 static int rt_acct_proc_show(struct seq_file *m, void *v) 342 { 343 struct ip_rt_acct *dst, *src; 344 unsigned int i, j; 345 346 dst = kcalloc(256, sizeof(struct ip_rt_acct), GFP_KERNEL); 347 if (!dst) 348 return -ENOMEM; 349 350 for_each_possible_cpu(i) { 351 src = (struct ip_rt_acct *)per_cpu_ptr(ip_rt_acct, i); 352 for (j = 0; j < 256; j++) { 353 dst[j].o_bytes += src[j].o_bytes; 354 dst[j].o_packets += src[j].o_packets; 355 dst[j].i_bytes += src[j].i_bytes; 356 dst[j].i_packets += src[j].i_packets; 357 } 358 } 359 360 seq_write(m, dst, 256 * sizeof(struct ip_rt_acct)); 361 kfree(dst); 362 return 0; 363 } 364 #endif 365 366 static int __net_init ip_rt_do_proc_init(struct net *net) 367 { 368 struct proc_dir_entry *pde; 369 370 pde = proc_create("rt_cache", 0444, net->proc_net, 371 &rt_cache_proc_ops); 372 if (!pde) 373 goto err1; 374 375 pde = proc_create("rt_cache", 0444, 376 net->proc_net_stat, &rt_cpu_proc_ops); 377 if (!pde) 378 goto err2; 379 380 #ifdef CONFIG_IP_ROUTE_CLASSID 381 pde = proc_create_single("rt_acct", 0, net->proc_net, 382 rt_acct_proc_show); 383 if (!pde) 384 goto err3; 385 #endif 386 return 0; 387 388 #ifdef CONFIG_IP_ROUTE_CLASSID 389 err3: 390 remove_proc_entry("rt_cache", net->proc_net_stat); 391 #endif 392 err2: 393 remove_proc_entry("rt_cache", net->proc_net); 394 err1: 395 return -ENOMEM; 396 } 397 398 static void __net_exit ip_rt_do_proc_exit(struct net *net) 399 { 400 remove_proc_entry("rt_cache", net->proc_net_stat); 401 remove_proc_entry("rt_cache", net->proc_net); 402 #ifdef CONFIG_IP_ROUTE_CLASSID 403 remove_proc_entry("rt_acct", net->proc_net); 404 #endif 405 } 406 407 static struct pernet_operations ip_rt_proc_ops __net_initdata = { 408 .init = ip_rt_do_proc_init, 409 .exit = ip_rt_do_proc_exit, 410 }; 411 412 static int __init ip_rt_proc_init(void) 413 { 414 return register_pernet_subsys(&ip_rt_proc_ops); 415 } 416 417 #else 418 static inline int ip_rt_proc_init(void) 419 { 420 return 0; 421 } 422 #endif /* CONFIG_PROC_FS */ 423 424 static inline bool rt_is_expired(const struct rtable *rth) 425 { 426 return rth->rt_genid != rt_genid_ipv4(dev_net(rth->dst.dev)); 427 } 428 429 void rt_cache_flush(struct net *net) 430 { 431 rt_genid_bump_ipv4(net); 432 } 433 434 static struct neighbour *ipv4_neigh_lookup(const struct dst_entry *dst, 435 struct sk_buff *skb, 436 const void *daddr) 437 { 438 const struct rtable *rt = container_of(dst, struct rtable, dst); 439 struct net_device *dev = dst->dev; 440 struct neighbour *n; 441 442 rcu_read_lock_bh(); 443 444 if (likely(rt->rt_gw_family == AF_INET)) { 445 n = ip_neigh_gw4(dev, rt->rt_gw4); 446 } else if (rt->rt_gw_family == AF_INET6) { 447 n = ip_neigh_gw6(dev, &rt->rt_gw6); 448 } else { 449 __be32 pkey; 450 451 pkey = skb ? ip_hdr(skb)->daddr : *((__be32 *) daddr); 452 n = ip_neigh_gw4(dev, pkey); 453 } 454 455 if (!IS_ERR(n) && !refcount_inc_not_zero(&n->refcnt)) 456 n = NULL; 457 458 rcu_read_unlock_bh(); 459 460 return n; 461 } 462 463 static void ipv4_confirm_neigh(const struct dst_entry *dst, const void *daddr) 464 { 465 const struct rtable *rt = container_of(dst, struct rtable, dst); 466 struct net_device *dev = dst->dev; 467 const __be32 *pkey = daddr; 468 469 if (rt->rt_gw_family == AF_INET) { 470 pkey = (const __be32 *)&rt->rt_gw4; 471 } else if (rt->rt_gw_family == AF_INET6) { 472 return __ipv6_confirm_neigh_stub(dev, &rt->rt_gw6); 473 } else if (!daddr || 474 (rt->rt_flags & 475 (RTCF_MULTICAST | RTCF_BROADCAST | RTCF_LOCAL))) { 476 return; 477 } 478 __ipv4_confirm_neigh(dev, *(__force u32 *)pkey); 479 } 480 481 #define IP_IDENTS_SZ 2048u 482 483 static atomic_t *ip_idents __read_mostly; 484 static u32 *ip_tstamps __read_mostly; 485 486 /* In order to protect privacy, we add a perturbation to identifiers 487 * if one generator is seldom used. This makes hard for an attacker 488 * to infer how many packets were sent between two points in time. 489 */ 490 u32 ip_idents_reserve(u32 hash, int segs) 491 { 492 u32 *p_tstamp = ip_tstamps + hash % IP_IDENTS_SZ; 493 atomic_t *p_id = ip_idents + hash % IP_IDENTS_SZ; 494 u32 old = READ_ONCE(*p_tstamp); 495 u32 now = (u32)jiffies; 496 u32 delta = 0; 497 498 if (old != now && cmpxchg(p_tstamp, old, now) == old) 499 delta = prandom_u32_max(now - old); 500 501 /* If UBSAN reports an error there, please make sure your compiler 502 * supports -fno-strict-overflow before reporting it that was a bug 503 * in UBSAN, and it has been fixed in GCC-8. 504 */ 505 return atomic_add_return(segs + delta, p_id) - segs; 506 } 507 EXPORT_SYMBOL(ip_idents_reserve); 508 509 void __ip_select_ident(struct net *net, struct iphdr *iph, int segs) 510 { 511 u32 hash, id; 512 513 /* Note the following code is not safe, but this is okay. */ 514 if (unlikely(siphash_key_is_zero(&net->ipv4.ip_id_key))) 515 get_random_bytes(&net->ipv4.ip_id_key, 516 sizeof(net->ipv4.ip_id_key)); 517 518 hash = siphash_3u32((__force u32)iph->daddr, 519 (__force u32)iph->saddr, 520 iph->protocol, 521 &net->ipv4.ip_id_key); 522 id = ip_idents_reserve(hash, segs); 523 iph->id = htons(id); 524 } 525 EXPORT_SYMBOL(__ip_select_ident); 526 527 static void __build_flow_key(const struct net *net, struct flowi4 *fl4, 528 const struct sock *sk, 529 const struct iphdr *iph, 530 int oif, u8 tos, 531 u8 prot, u32 mark, int flow_flags) 532 { 533 if (sk) { 534 const struct inet_sock *inet = inet_sk(sk); 535 536 oif = sk->sk_bound_dev_if; 537 mark = sk->sk_mark; 538 tos = RT_CONN_FLAGS(sk); 539 prot = inet->hdrincl ? IPPROTO_RAW : sk->sk_protocol; 540 } 541 flowi4_init_output(fl4, oif, mark, tos, 542 RT_SCOPE_UNIVERSE, prot, 543 flow_flags, 544 iph->daddr, iph->saddr, 0, 0, 545 sock_net_uid(net, sk)); 546 } 547 548 static void build_skb_flow_key(struct flowi4 *fl4, const struct sk_buff *skb, 549 const struct sock *sk) 550 { 551 const struct net *net = dev_net(skb->dev); 552 const struct iphdr *iph = ip_hdr(skb); 553 int oif = skb->dev->ifindex; 554 u8 tos = RT_TOS(iph->tos); 555 u8 prot = iph->protocol; 556 u32 mark = skb->mark; 557 558 __build_flow_key(net, fl4, sk, iph, oif, tos, prot, mark, 0); 559 } 560 561 static void build_sk_flow_key(struct flowi4 *fl4, const struct sock *sk) 562 { 563 const struct inet_sock *inet = inet_sk(sk); 564 const struct ip_options_rcu *inet_opt; 565 __be32 daddr = inet->inet_daddr; 566 567 rcu_read_lock(); 568 inet_opt = rcu_dereference(inet->inet_opt); 569 if (inet_opt && inet_opt->opt.srr) 570 daddr = inet_opt->opt.faddr; 571 flowi4_init_output(fl4, sk->sk_bound_dev_if, sk->sk_mark, 572 RT_CONN_FLAGS(sk), RT_SCOPE_UNIVERSE, 573 inet->hdrincl ? IPPROTO_RAW : sk->sk_protocol, 574 inet_sk_flowi_flags(sk), 575 daddr, inet->inet_saddr, 0, 0, sk->sk_uid); 576 rcu_read_unlock(); 577 } 578 579 static void ip_rt_build_flow_key(struct flowi4 *fl4, const struct sock *sk, 580 const struct sk_buff *skb) 581 { 582 if (skb) 583 build_skb_flow_key(fl4, skb, sk); 584 else 585 build_sk_flow_key(fl4, sk); 586 } 587 588 static DEFINE_SPINLOCK(fnhe_lock); 589 590 static void fnhe_flush_routes(struct fib_nh_exception *fnhe) 591 { 592 struct rtable *rt; 593 594 rt = rcu_dereference(fnhe->fnhe_rth_input); 595 if (rt) { 596 RCU_INIT_POINTER(fnhe->fnhe_rth_input, NULL); 597 dst_dev_put(&rt->dst); 598 dst_release(&rt->dst); 599 } 600 rt = rcu_dereference(fnhe->fnhe_rth_output); 601 if (rt) { 602 RCU_INIT_POINTER(fnhe->fnhe_rth_output, NULL); 603 dst_dev_put(&rt->dst); 604 dst_release(&rt->dst); 605 } 606 } 607 608 static struct fib_nh_exception *fnhe_oldest(struct fnhe_hash_bucket *hash) 609 { 610 struct fib_nh_exception *fnhe, *oldest; 611 612 oldest = rcu_dereference(hash->chain); 613 for (fnhe = rcu_dereference(oldest->fnhe_next); fnhe; 614 fnhe = rcu_dereference(fnhe->fnhe_next)) { 615 if (time_before(fnhe->fnhe_stamp, oldest->fnhe_stamp)) 616 oldest = fnhe; 617 } 618 fnhe_flush_routes(oldest); 619 return oldest; 620 } 621 622 static inline u32 fnhe_hashfun(__be32 daddr) 623 { 624 static u32 fnhe_hashrnd __read_mostly; 625 u32 hval; 626 627 net_get_random_once(&fnhe_hashrnd, sizeof(fnhe_hashrnd)); 628 hval = jhash_1word((__force u32)daddr, fnhe_hashrnd); 629 return hash_32(hval, FNHE_HASH_SHIFT); 630 } 631 632 static void fill_route_from_fnhe(struct rtable *rt, struct fib_nh_exception *fnhe) 633 { 634 rt->rt_pmtu = fnhe->fnhe_pmtu; 635 rt->rt_mtu_locked = fnhe->fnhe_mtu_locked; 636 rt->dst.expires = fnhe->fnhe_expires; 637 638 if (fnhe->fnhe_gw) { 639 rt->rt_flags |= RTCF_REDIRECTED; 640 rt->rt_uses_gateway = 1; 641 rt->rt_gw_family = AF_INET; 642 rt->rt_gw4 = fnhe->fnhe_gw; 643 } 644 } 645 646 static void update_or_create_fnhe(struct fib_nh_common *nhc, __be32 daddr, 647 __be32 gw, u32 pmtu, bool lock, 648 unsigned long expires) 649 { 650 struct fnhe_hash_bucket *hash; 651 struct fib_nh_exception *fnhe; 652 struct rtable *rt; 653 u32 genid, hval; 654 unsigned int i; 655 int depth; 656 657 genid = fnhe_genid(dev_net(nhc->nhc_dev)); 658 hval = fnhe_hashfun(daddr); 659 660 spin_lock_bh(&fnhe_lock); 661 662 hash = rcu_dereference(nhc->nhc_exceptions); 663 if (!hash) { 664 hash = kcalloc(FNHE_HASH_SIZE, sizeof(*hash), GFP_ATOMIC); 665 if (!hash) 666 goto out_unlock; 667 rcu_assign_pointer(nhc->nhc_exceptions, hash); 668 } 669 670 hash += hval; 671 672 depth = 0; 673 for (fnhe = rcu_dereference(hash->chain); fnhe; 674 fnhe = rcu_dereference(fnhe->fnhe_next)) { 675 if (fnhe->fnhe_daddr == daddr) 676 break; 677 depth++; 678 } 679 680 if (fnhe) { 681 if (fnhe->fnhe_genid != genid) 682 fnhe->fnhe_genid = genid; 683 if (gw) 684 fnhe->fnhe_gw = gw; 685 if (pmtu) { 686 fnhe->fnhe_pmtu = pmtu; 687 fnhe->fnhe_mtu_locked = lock; 688 } 689 fnhe->fnhe_expires = max(1UL, expires); 690 /* Update all cached dsts too */ 691 rt = rcu_dereference(fnhe->fnhe_rth_input); 692 if (rt) 693 fill_route_from_fnhe(rt, fnhe); 694 rt = rcu_dereference(fnhe->fnhe_rth_output); 695 if (rt) 696 fill_route_from_fnhe(rt, fnhe); 697 } else { 698 if (depth > FNHE_RECLAIM_DEPTH) 699 fnhe = fnhe_oldest(hash); 700 else { 701 fnhe = kzalloc(sizeof(*fnhe), GFP_ATOMIC); 702 if (!fnhe) 703 goto out_unlock; 704 705 fnhe->fnhe_next = hash->chain; 706 rcu_assign_pointer(hash->chain, fnhe); 707 } 708 fnhe->fnhe_genid = genid; 709 fnhe->fnhe_daddr = daddr; 710 fnhe->fnhe_gw = gw; 711 fnhe->fnhe_pmtu = pmtu; 712 fnhe->fnhe_mtu_locked = lock; 713 fnhe->fnhe_expires = max(1UL, expires); 714 715 /* Exception created; mark the cached routes for the nexthop 716 * stale, so anyone caching it rechecks if this exception 717 * applies to them. 718 */ 719 rt = rcu_dereference(nhc->nhc_rth_input); 720 if (rt) 721 rt->dst.obsolete = DST_OBSOLETE_KILL; 722 723 for_each_possible_cpu(i) { 724 struct rtable __rcu **prt; 725 726 prt = per_cpu_ptr(nhc->nhc_pcpu_rth_output, i); 727 rt = rcu_dereference(*prt); 728 if (rt) 729 rt->dst.obsolete = DST_OBSOLETE_KILL; 730 } 731 } 732 733 fnhe->fnhe_stamp = jiffies; 734 735 out_unlock: 736 spin_unlock_bh(&fnhe_lock); 737 } 738 739 static void __ip_do_redirect(struct rtable *rt, struct sk_buff *skb, struct flowi4 *fl4, 740 bool kill_route) 741 { 742 __be32 new_gw = icmp_hdr(skb)->un.gateway; 743 __be32 old_gw = ip_hdr(skb)->saddr; 744 struct net_device *dev = skb->dev; 745 struct in_device *in_dev; 746 struct fib_result res; 747 struct neighbour *n; 748 struct net *net; 749 750 switch (icmp_hdr(skb)->code & 7) { 751 case ICMP_REDIR_NET: 752 case ICMP_REDIR_NETTOS: 753 case ICMP_REDIR_HOST: 754 case ICMP_REDIR_HOSTTOS: 755 break; 756 757 default: 758 return; 759 } 760 761 if (rt->rt_gw_family != AF_INET || rt->rt_gw4 != old_gw) 762 return; 763 764 in_dev = __in_dev_get_rcu(dev); 765 if (!in_dev) 766 return; 767 768 net = dev_net(dev); 769 if (new_gw == old_gw || !IN_DEV_RX_REDIRECTS(in_dev) || 770 ipv4_is_multicast(new_gw) || ipv4_is_lbcast(new_gw) || 771 ipv4_is_zeronet(new_gw)) 772 goto reject_redirect; 773 774 if (!IN_DEV_SHARED_MEDIA(in_dev)) { 775 if (!inet_addr_onlink(in_dev, new_gw, old_gw)) 776 goto reject_redirect; 777 if (IN_DEV_SEC_REDIRECTS(in_dev) && ip_fib_check_default(new_gw, dev)) 778 goto reject_redirect; 779 } else { 780 if (inet_addr_type(net, new_gw) != RTN_UNICAST) 781 goto reject_redirect; 782 } 783 784 n = __ipv4_neigh_lookup(rt->dst.dev, new_gw); 785 if (!n) 786 n = neigh_create(&arp_tbl, &new_gw, rt->dst.dev); 787 if (!IS_ERR(n)) { 788 if (!(n->nud_state & NUD_VALID)) { 789 neigh_event_send(n, NULL); 790 } else { 791 if (fib_lookup(net, fl4, &res, 0) == 0) { 792 struct fib_nh_common *nhc; 793 794 fib_select_path(net, &res, fl4, skb); 795 nhc = FIB_RES_NHC(res); 796 update_or_create_fnhe(nhc, fl4->daddr, new_gw, 797 0, false, 798 jiffies + ip_rt_gc_timeout); 799 } 800 if (kill_route) 801 rt->dst.obsolete = DST_OBSOLETE_KILL; 802 call_netevent_notifiers(NETEVENT_NEIGH_UPDATE, n); 803 } 804 neigh_release(n); 805 } 806 return; 807 808 reject_redirect: 809 #ifdef CONFIG_IP_ROUTE_VERBOSE 810 if (IN_DEV_LOG_MARTIANS(in_dev)) { 811 const struct iphdr *iph = (const struct iphdr *) skb->data; 812 __be32 daddr = iph->daddr; 813 __be32 saddr = iph->saddr; 814 815 net_info_ratelimited("Redirect from %pI4 on %s about %pI4 ignored\n" 816 " Advised path = %pI4 -> %pI4\n", 817 &old_gw, dev->name, &new_gw, 818 &saddr, &daddr); 819 } 820 #endif 821 ; 822 } 823 824 static void ip_do_redirect(struct dst_entry *dst, struct sock *sk, struct sk_buff *skb) 825 { 826 struct rtable *rt; 827 struct flowi4 fl4; 828 const struct iphdr *iph = (const struct iphdr *) skb->data; 829 struct net *net = dev_net(skb->dev); 830 int oif = skb->dev->ifindex; 831 u8 tos = RT_TOS(iph->tos); 832 u8 prot = iph->protocol; 833 u32 mark = skb->mark; 834 835 rt = (struct rtable *) dst; 836 837 __build_flow_key(net, &fl4, sk, iph, oif, tos, prot, mark, 0); 838 __ip_do_redirect(rt, skb, &fl4, true); 839 } 840 841 static struct dst_entry *ipv4_negative_advice(struct dst_entry *dst) 842 { 843 struct rtable *rt = (struct rtable *)dst; 844 struct dst_entry *ret = dst; 845 846 if (rt) { 847 if (dst->obsolete > 0) { 848 ip_rt_put(rt); 849 ret = NULL; 850 } else if ((rt->rt_flags & RTCF_REDIRECTED) || 851 rt->dst.expires) { 852 ip_rt_put(rt); 853 ret = NULL; 854 } 855 } 856 return ret; 857 } 858 859 /* 860 * Algorithm: 861 * 1. The first ip_rt_redirect_number redirects are sent 862 * with exponential backoff, then we stop sending them at all, 863 * assuming that the host ignores our redirects. 864 * 2. If we did not see packets requiring redirects 865 * during ip_rt_redirect_silence, we assume that the host 866 * forgot redirected route and start to send redirects again. 867 * 868 * This algorithm is much cheaper and more intelligent than dumb load limiting 869 * in icmp.c. 870 * 871 * NOTE. Do not forget to inhibit load limiting for redirects (redundant) 872 * and "frag. need" (breaks PMTU discovery) in icmp.c. 873 */ 874 875 void ip_rt_send_redirect(struct sk_buff *skb) 876 { 877 struct rtable *rt = skb_rtable(skb); 878 struct in_device *in_dev; 879 struct inet_peer *peer; 880 struct net *net; 881 int log_martians; 882 int vif; 883 884 rcu_read_lock(); 885 in_dev = __in_dev_get_rcu(rt->dst.dev); 886 if (!in_dev || !IN_DEV_TX_REDIRECTS(in_dev)) { 887 rcu_read_unlock(); 888 return; 889 } 890 log_martians = IN_DEV_LOG_MARTIANS(in_dev); 891 vif = l3mdev_master_ifindex_rcu(rt->dst.dev); 892 rcu_read_unlock(); 893 894 net = dev_net(rt->dst.dev); 895 peer = inet_getpeer_v4(net->ipv4.peers, ip_hdr(skb)->saddr, vif, 1); 896 if (!peer) { 897 icmp_send(skb, ICMP_REDIRECT, ICMP_REDIR_HOST, 898 rt_nexthop(rt, ip_hdr(skb)->daddr)); 899 return; 900 } 901 902 /* No redirected packets during ip_rt_redirect_silence; 903 * reset the algorithm. 904 */ 905 if (time_after(jiffies, peer->rate_last + ip_rt_redirect_silence)) { 906 peer->rate_tokens = 0; 907 peer->n_redirects = 0; 908 } 909 910 /* Too many ignored redirects; do not send anything 911 * set dst.rate_last to the last seen redirected packet. 912 */ 913 if (peer->n_redirects >= ip_rt_redirect_number) { 914 peer->rate_last = jiffies; 915 goto out_put_peer; 916 } 917 918 /* Check for load limit; set rate_last to the latest sent 919 * redirect. 920 */ 921 if (peer->n_redirects == 0 || 922 time_after(jiffies, 923 (peer->rate_last + 924 (ip_rt_redirect_load << peer->n_redirects)))) { 925 __be32 gw = rt_nexthop(rt, ip_hdr(skb)->daddr); 926 927 icmp_send(skb, ICMP_REDIRECT, ICMP_REDIR_HOST, gw); 928 peer->rate_last = jiffies; 929 ++peer->n_redirects; 930 #ifdef CONFIG_IP_ROUTE_VERBOSE 931 if (log_martians && 932 peer->n_redirects == ip_rt_redirect_number) 933 net_warn_ratelimited("host %pI4/if%d ignores redirects for %pI4 to %pI4\n", 934 &ip_hdr(skb)->saddr, inet_iif(skb), 935 &ip_hdr(skb)->daddr, &gw); 936 #endif 937 } 938 out_put_peer: 939 inet_putpeer(peer); 940 } 941 942 static int ip_error(struct sk_buff *skb) 943 { 944 struct rtable *rt = skb_rtable(skb); 945 struct net_device *dev = skb->dev; 946 struct in_device *in_dev; 947 struct inet_peer *peer; 948 unsigned long now; 949 struct net *net; 950 bool send; 951 int code; 952 953 if (netif_is_l3_master(skb->dev)) { 954 dev = __dev_get_by_index(dev_net(skb->dev), IPCB(skb)->iif); 955 if (!dev) 956 goto out; 957 } 958 959 in_dev = __in_dev_get_rcu(dev); 960 961 /* IP on this device is disabled. */ 962 if (!in_dev) 963 goto out; 964 965 net = dev_net(rt->dst.dev); 966 if (!IN_DEV_FORWARD(in_dev)) { 967 switch (rt->dst.error) { 968 case EHOSTUNREACH: 969 __IP_INC_STATS(net, IPSTATS_MIB_INADDRERRORS); 970 break; 971 972 case ENETUNREACH: 973 __IP_INC_STATS(net, IPSTATS_MIB_INNOROUTES); 974 break; 975 } 976 goto out; 977 } 978 979 switch (rt->dst.error) { 980 case EINVAL: 981 default: 982 goto out; 983 case EHOSTUNREACH: 984 code = ICMP_HOST_UNREACH; 985 break; 986 case ENETUNREACH: 987 code = ICMP_NET_UNREACH; 988 __IP_INC_STATS(net, IPSTATS_MIB_INNOROUTES); 989 break; 990 case EACCES: 991 code = ICMP_PKT_FILTERED; 992 break; 993 } 994 995 peer = inet_getpeer_v4(net->ipv4.peers, ip_hdr(skb)->saddr, 996 l3mdev_master_ifindex(skb->dev), 1); 997 998 send = true; 999 if (peer) { 1000 now = jiffies; 1001 peer->rate_tokens += now - peer->rate_last; 1002 if (peer->rate_tokens > ip_rt_error_burst) 1003 peer->rate_tokens = ip_rt_error_burst; 1004 peer->rate_last = now; 1005 if (peer->rate_tokens >= ip_rt_error_cost) 1006 peer->rate_tokens -= ip_rt_error_cost; 1007 else 1008 send = false; 1009 inet_putpeer(peer); 1010 } 1011 if (send) 1012 icmp_send(skb, ICMP_DEST_UNREACH, code, 0); 1013 1014 out: kfree_skb(skb); 1015 return 0; 1016 } 1017 1018 static void __ip_rt_update_pmtu(struct rtable *rt, struct flowi4 *fl4, u32 mtu) 1019 { 1020 struct dst_entry *dst = &rt->dst; 1021 struct net *net = dev_net(dst->dev); 1022 struct fib_result res; 1023 bool lock = false; 1024 u32 old_mtu; 1025 1026 if (ip_mtu_locked(dst)) 1027 return; 1028 1029 old_mtu = ipv4_mtu(dst); 1030 if (old_mtu < mtu) 1031 return; 1032 1033 if (mtu < ip_rt_min_pmtu) { 1034 lock = true; 1035 mtu = min(old_mtu, ip_rt_min_pmtu); 1036 } 1037 1038 if (rt->rt_pmtu == mtu && !lock && 1039 time_before(jiffies, dst->expires - ip_rt_mtu_expires / 2)) 1040 return; 1041 1042 rcu_read_lock(); 1043 if (fib_lookup(net, fl4, &res, 0) == 0) { 1044 struct fib_nh_common *nhc; 1045 1046 fib_select_path(net, &res, fl4, NULL); 1047 nhc = FIB_RES_NHC(res); 1048 update_or_create_fnhe(nhc, fl4->daddr, 0, mtu, lock, 1049 jiffies + ip_rt_mtu_expires); 1050 } 1051 rcu_read_unlock(); 1052 } 1053 1054 static void ip_rt_update_pmtu(struct dst_entry *dst, struct sock *sk, 1055 struct sk_buff *skb, u32 mtu, 1056 bool confirm_neigh) 1057 { 1058 struct rtable *rt = (struct rtable *) dst; 1059 struct flowi4 fl4; 1060 1061 ip_rt_build_flow_key(&fl4, sk, skb); 1062 1063 /* Don't make lookup fail for bridged encapsulations */ 1064 if (skb && netif_is_any_bridge_port(skb->dev)) 1065 fl4.flowi4_oif = 0; 1066 1067 __ip_rt_update_pmtu(rt, &fl4, mtu); 1068 } 1069 1070 void ipv4_update_pmtu(struct sk_buff *skb, struct net *net, u32 mtu, 1071 int oif, u8 protocol) 1072 { 1073 const struct iphdr *iph = (const struct iphdr *)skb->data; 1074 struct flowi4 fl4; 1075 struct rtable *rt; 1076 u32 mark = IP4_REPLY_MARK(net, skb->mark); 1077 1078 __build_flow_key(net, &fl4, NULL, iph, oif, 1079 RT_TOS(iph->tos), protocol, mark, 0); 1080 rt = __ip_route_output_key(net, &fl4); 1081 if (!IS_ERR(rt)) { 1082 __ip_rt_update_pmtu(rt, &fl4, mtu); 1083 ip_rt_put(rt); 1084 } 1085 } 1086 EXPORT_SYMBOL_GPL(ipv4_update_pmtu); 1087 1088 static void __ipv4_sk_update_pmtu(struct sk_buff *skb, struct sock *sk, u32 mtu) 1089 { 1090 const struct iphdr *iph = (const struct iphdr *)skb->data; 1091 struct flowi4 fl4; 1092 struct rtable *rt; 1093 1094 __build_flow_key(sock_net(sk), &fl4, sk, iph, 0, 0, 0, 0, 0); 1095 1096 if (!fl4.flowi4_mark) 1097 fl4.flowi4_mark = IP4_REPLY_MARK(sock_net(sk), skb->mark); 1098 1099 rt = __ip_route_output_key(sock_net(sk), &fl4); 1100 if (!IS_ERR(rt)) { 1101 __ip_rt_update_pmtu(rt, &fl4, mtu); 1102 ip_rt_put(rt); 1103 } 1104 } 1105 1106 void ipv4_sk_update_pmtu(struct sk_buff *skb, struct sock *sk, u32 mtu) 1107 { 1108 const struct iphdr *iph = (const struct iphdr *)skb->data; 1109 struct flowi4 fl4; 1110 struct rtable *rt; 1111 struct dst_entry *odst = NULL; 1112 bool new = false; 1113 struct net *net = sock_net(sk); 1114 1115 bh_lock_sock(sk); 1116 1117 if (!ip_sk_accept_pmtu(sk)) 1118 goto out; 1119 1120 odst = sk_dst_get(sk); 1121 1122 if (sock_owned_by_user(sk) || !odst) { 1123 __ipv4_sk_update_pmtu(skb, sk, mtu); 1124 goto out; 1125 } 1126 1127 __build_flow_key(net, &fl4, sk, iph, 0, 0, 0, 0, 0); 1128 1129 rt = (struct rtable *)odst; 1130 if (odst->obsolete && !odst->ops->check(odst, 0)) { 1131 rt = ip_route_output_flow(sock_net(sk), &fl4, sk); 1132 if (IS_ERR(rt)) 1133 goto out; 1134 1135 new = true; 1136 } 1137 1138 __ip_rt_update_pmtu((struct rtable *)xfrm_dst_path(&rt->dst), &fl4, mtu); 1139 1140 if (!dst_check(&rt->dst, 0)) { 1141 if (new) 1142 dst_release(&rt->dst); 1143 1144 rt = ip_route_output_flow(sock_net(sk), &fl4, sk); 1145 if (IS_ERR(rt)) 1146 goto out; 1147 1148 new = true; 1149 } 1150 1151 if (new) 1152 sk_dst_set(sk, &rt->dst); 1153 1154 out: 1155 bh_unlock_sock(sk); 1156 dst_release(odst); 1157 } 1158 EXPORT_SYMBOL_GPL(ipv4_sk_update_pmtu); 1159 1160 void ipv4_redirect(struct sk_buff *skb, struct net *net, 1161 int oif, u8 protocol) 1162 { 1163 const struct iphdr *iph = (const struct iphdr *)skb->data; 1164 struct flowi4 fl4; 1165 struct rtable *rt; 1166 1167 __build_flow_key(net, &fl4, NULL, iph, oif, 1168 RT_TOS(iph->tos), protocol, 0, 0); 1169 rt = __ip_route_output_key(net, &fl4); 1170 if (!IS_ERR(rt)) { 1171 __ip_do_redirect(rt, skb, &fl4, false); 1172 ip_rt_put(rt); 1173 } 1174 } 1175 EXPORT_SYMBOL_GPL(ipv4_redirect); 1176 1177 void ipv4_sk_redirect(struct sk_buff *skb, struct sock *sk) 1178 { 1179 const struct iphdr *iph = (const struct iphdr *)skb->data; 1180 struct flowi4 fl4; 1181 struct rtable *rt; 1182 struct net *net = sock_net(sk); 1183 1184 __build_flow_key(net, &fl4, sk, iph, 0, 0, 0, 0, 0); 1185 rt = __ip_route_output_key(net, &fl4); 1186 if (!IS_ERR(rt)) { 1187 __ip_do_redirect(rt, skb, &fl4, false); 1188 ip_rt_put(rt); 1189 } 1190 } 1191 EXPORT_SYMBOL_GPL(ipv4_sk_redirect); 1192 1193 INDIRECT_CALLABLE_SCOPE struct dst_entry *ipv4_dst_check(struct dst_entry *dst, 1194 u32 cookie) 1195 { 1196 struct rtable *rt = (struct rtable *) dst; 1197 1198 /* All IPV4 dsts are created with ->obsolete set to the value 1199 * DST_OBSOLETE_FORCE_CHK which forces validation calls down 1200 * into this function always. 1201 * 1202 * When a PMTU/redirect information update invalidates a route, 1203 * this is indicated by setting obsolete to DST_OBSOLETE_KILL or 1204 * DST_OBSOLETE_DEAD. 1205 */ 1206 if (dst->obsolete != DST_OBSOLETE_FORCE_CHK || rt_is_expired(rt)) 1207 return NULL; 1208 return dst; 1209 } 1210 EXPORT_INDIRECT_CALLABLE(ipv4_dst_check); 1211 1212 static void ipv4_send_dest_unreach(struct sk_buff *skb) 1213 { 1214 struct ip_options opt; 1215 int res; 1216 1217 /* Recompile ip options since IPCB may not be valid anymore. 1218 * Also check we have a reasonable ipv4 header. 1219 */ 1220 if (!pskb_network_may_pull(skb, sizeof(struct iphdr)) || 1221 ip_hdr(skb)->version != 4 || ip_hdr(skb)->ihl < 5) 1222 return; 1223 1224 memset(&opt, 0, sizeof(opt)); 1225 if (ip_hdr(skb)->ihl > 5) { 1226 if (!pskb_network_may_pull(skb, ip_hdr(skb)->ihl * 4)) 1227 return; 1228 opt.optlen = ip_hdr(skb)->ihl * 4 - sizeof(struct iphdr); 1229 1230 rcu_read_lock(); 1231 res = __ip_options_compile(dev_net(skb->dev), &opt, skb, NULL); 1232 rcu_read_unlock(); 1233 1234 if (res) 1235 return; 1236 } 1237 __icmp_send(skb, ICMP_DEST_UNREACH, ICMP_HOST_UNREACH, 0, &opt); 1238 } 1239 1240 static void ipv4_link_failure(struct sk_buff *skb) 1241 { 1242 struct rtable *rt; 1243 1244 ipv4_send_dest_unreach(skb); 1245 1246 rt = skb_rtable(skb); 1247 if (rt) 1248 dst_set_expires(&rt->dst, 0); 1249 } 1250 1251 static int ip_rt_bug(struct net *net, struct sock *sk, struct sk_buff *skb) 1252 { 1253 pr_debug("%s: %pI4 -> %pI4, %s\n", 1254 __func__, &ip_hdr(skb)->saddr, &ip_hdr(skb)->daddr, 1255 skb->dev ? skb->dev->name : "?"); 1256 kfree_skb(skb); 1257 WARN_ON(1); 1258 return 0; 1259 } 1260 1261 /* 1262 * We do not cache source address of outgoing interface, 1263 * because it is used only by IP RR, TS and SRR options, 1264 * so that it out of fast path. 1265 * 1266 * BTW remember: "addr" is allowed to be not aligned 1267 * in IP options! 1268 */ 1269 1270 void ip_rt_get_source(u8 *addr, struct sk_buff *skb, struct rtable *rt) 1271 { 1272 __be32 src; 1273 1274 if (rt_is_output_route(rt)) 1275 src = ip_hdr(skb)->saddr; 1276 else { 1277 struct fib_result res; 1278 struct iphdr *iph = ip_hdr(skb); 1279 struct flowi4 fl4 = { 1280 .daddr = iph->daddr, 1281 .saddr = iph->saddr, 1282 .flowi4_tos = RT_TOS(iph->tos), 1283 .flowi4_oif = rt->dst.dev->ifindex, 1284 .flowi4_iif = skb->dev->ifindex, 1285 .flowi4_mark = skb->mark, 1286 }; 1287 1288 rcu_read_lock(); 1289 if (fib_lookup(dev_net(rt->dst.dev), &fl4, &res, 0) == 0) 1290 src = fib_result_prefsrc(dev_net(rt->dst.dev), &res); 1291 else 1292 src = inet_select_addr(rt->dst.dev, 1293 rt_nexthop(rt, iph->daddr), 1294 RT_SCOPE_UNIVERSE); 1295 rcu_read_unlock(); 1296 } 1297 memcpy(addr, &src, 4); 1298 } 1299 1300 #ifdef CONFIG_IP_ROUTE_CLASSID 1301 static void set_class_tag(struct rtable *rt, u32 tag) 1302 { 1303 if (!(rt->dst.tclassid & 0xFFFF)) 1304 rt->dst.tclassid |= tag & 0xFFFF; 1305 if (!(rt->dst.tclassid & 0xFFFF0000)) 1306 rt->dst.tclassid |= tag & 0xFFFF0000; 1307 } 1308 #endif 1309 1310 static unsigned int ipv4_default_advmss(const struct dst_entry *dst) 1311 { 1312 unsigned int header_size = sizeof(struct tcphdr) + sizeof(struct iphdr); 1313 unsigned int advmss = max_t(unsigned int, ipv4_mtu(dst) - header_size, 1314 ip_rt_min_advmss); 1315 1316 return min(advmss, IPV4_MAX_PMTU - header_size); 1317 } 1318 1319 INDIRECT_CALLABLE_SCOPE unsigned int ipv4_mtu(const struct dst_entry *dst) 1320 { 1321 const struct rtable *rt = (const struct rtable *)dst; 1322 unsigned int mtu = rt->rt_pmtu; 1323 1324 if (!mtu || time_after_eq(jiffies, rt->dst.expires)) 1325 mtu = dst_metric_raw(dst, RTAX_MTU); 1326 1327 if (mtu) 1328 return mtu; 1329 1330 mtu = READ_ONCE(dst->dev->mtu); 1331 1332 if (unlikely(ip_mtu_locked(dst))) { 1333 if (rt->rt_uses_gateway && mtu > 576) 1334 mtu = 576; 1335 } 1336 1337 mtu = min_t(unsigned int, mtu, IP_MAX_MTU); 1338 1339 return mtu - lwtunnel_headroom(dst->lwtstate, mtu); 1340 } 1341 EXPORT_INDIRECT_CALLABLE(ipv4_mtu); 1342 1343 static void ip_del_fnhe(struct fib_nh_common *nhc, __be32 daddr) 1344 { 1345 struct fnhe_hash_bucket *hash; 1346 struct fib_nh_exception *fnhe, __rcu **fnhe_p; 1347 u32 hval = fnhe_hashfun(daddr); 1348 1349 spin_lock_bh(&fnhe_lock); 1350 1351 hash = rcu_dereference_protected(nhc->nhc_exceptions, 1352 lockdep_is_held(&fnhe_lock)); 1353 hash += hval; 1354 1355 fnhe_p = &hash->chain; 1356 fnhe = rcu_dereference_protected(*fnhe_p, lockdep_is_held(&fnhe_lock)); 1357 while (fnhe) { 1358 if (fnhe->fnhe_daddr == daddr) { 1359 rcu_assign_pointer(*fnhe_p, rcu_dereference_protected( 1360 fnhe->fnhe_next, lockdep_is_held(&fnhe_lock))); 1361 /* set fnhe_daddr to 0 to ensure it won't bind with 1362 * new dsts in rt_bind_exception(). 1363 */ 1364 fnhe->fnhe_daddr = 0; 1365 fnhe_flush_routes(fnhe); 1366 kfree_rcu(fnhe, rcu); 1367 break; 1368 } 1369 fnhe_p = &fnhe->fnhe_next; 1370 fnhe = rcu_dereference_protected(fnhe->fnhe_next, 1371 lockdep_is_held(&fnhe_lock)); 1372 } 1373 1374 spin_unlock_bh(&fnhe_lock); 1375 } 1376 1377 static struct fib_nh_exception *find_exception(struct fib_nh_common *nhc, 1378 __be32 daddr) 1379 { 1380 struct fnhe_hash_bucket *hash = rcu_dereference(nhc->nhc_exceptions); 1381 struct fib_nh_exception *fnhe; 1382 u32 hval; 1383 1384 if (!hash) 1385 return NULL; 1386 1387 hval = fnhe_hashfun(daddr); 1388 1389 for (fnhe = rcu_dereference(hash[hval].chain); fnhe; 1390 fnhe = rcu_dereference(fnhe->fnhe_next)) { 1391 if (fnhe->fnhe_daddr == daddr) { 1392 if (fnhe->fnhe_expires && 1393 time_after(jiffies, fnhe->fnhe_expires)) { 1394 ip_del_fnhe(nhc, daddr); 1395 break; 1396 } 1397 return fnhe; 1398 } 1399 } 1400 return NULL; 1401 } 1402 1403 /* MTU selection: 1404 * 1. mtu on route is locked - use it 1405 * 2. mtu from nexthop exception 1406 * 3. mtu from egress device 1407 */ 1408 1409 u32 ip_mtu_from_fib_result(struct fib_result *res, __be32 daddr) 1410 { 1411 struct fib_nh_common *nhc = res->nhc; 1412 struct net_device *dev = nhc->nhc_dev; 1413 struct fib_info *fi = res->fi; 1414 u32 mtu = 0; 1415 1416 if (dev_net(dev)->ipv4.sysctl_ip_fwd_use_pmtu || 1417 fi->fib_metrics->metrics[RTAX_LOCK - 1] & (1 << RTAX_MTU)) 1418 mtu = fi->fib_mtu; 1419 1420 if (likely(!mtu)) { 1421 struct fib_nh_exception *fnhe; 1422 1423 fnhe = find_exception(nhc, daddr); 1424 if (fnhe && !time_after_eq(jiffies, fnhe->fnhe_expires)) 1425 mtu = fnhe->fnhe_pmtu; 1426 } 1427 1428 if (likely(!mtu)) 1429 mtu = min(READ_ONCE(dev->mtu), IP_MAX_MTU); 1430 1431 return mtu - lwtunnel_headroom(nhc->nhc_lwtstate, mtu); 1432 } 1433 1434 static bool rt_bind_exception(struct rtable *rt, struct fib_nh_exception *fnhe, 1435 __be32 daddr, const bool do_cache) 1436 { 1437 bool ret = false; 1438 1439 spin_lock_bh(&fnhe_lock); 1440 1441 if (daddr == fnhe->fnhe_daddr) { 1442 struct rtable __rcu **porig; 1443 struct rtable *orig; 1444 int genid = fnhe_genid(dev_net(rt->dst.dev)); 1445 1446 if (rt_is_input_route(rt)) 1447 porig = &fnhe->fnhe_rth_input; 1448 else 1449 porig = &fnhe->fnhe_rth_output; 1450 orig = rcu_dereference(*porig); 1451 1452 if (fnhe->fnhe_genid != genid) { 1453 fnhe->fnhe_genid = genid; 1454 fnhe->fnhe_gw = 0; 1455 fnhe->fnhe_pmtu = 0; 1456 fnhe->fnhe_expires = 0; 1457 fnhe->fnhe_mtu_locked = false; 1458 fnhe_flush_routes(fnhe); 1459 orig = NULL; 1460 } 1461 fill_route_from_fnhe(rt, fnhe); 1462 if (!rt->rt_gw4) { 1463 rt->rt_gw4 = daddr; 1464 rt->rt_gw_family = AF_INET; 1465 } 1466 1467 if (do_cache) { 1468 dst_hold(&rt->dst); 1469 rcu_assign_pointer(*porig, rt); 1470 if (orig) { 1471 dst_dev_put(&orig->dst); 1472 dst_release(&orig->dst); 1473 } 1474 ret = true; 1475 } 1476 1477 fnhe->fnhe_stamp = jiffies; 1478 } 1479 spin_unlock_bh(&fnhe_lock); 1480 1481 return ret; 1482 } 1483 1484 static bool rt_cache_route(struct fib_nh_common *nhc, struct rtable *rt) 1485 { 1486 struct rtable *orig, *prev, **p; 1487 bool ret = true; 1488 1489 if (rt_is_input_route(rt)) { 1490 p = (struct rtable **)&nhc->nhc_rth_input; 1491 } else { 1492 p = (struct rtable **)raw_cpu_ptr(nhc->nhc_pcpu_rth_output); 1493 } 1494 orig = *p; 1495 1496 /* hold dst before doing cmpxchg() to avoid race condition 1497 * on this dst 1498 */ 1499 dst_hold(&rt->dst); 1500 prev = cmpxchg(p, orig, rt); 1501 if (prev == orig) { 1502 if (orig) { 1503 rt_add_uncached_list(orig); 1504 dst_release(&orig->dst); 1505 } 1506 } else { 1507 dst_release(&rt->dst); 1508 ret = false; 1509 } 1510 1511 return ret; 1512 } 1513 1514 struct uncached_list { 1515 spinlock_t lock; 1516 struct list_head head; 1517 }; 1518 1519 static DEFINE_PER_CPU_ALIGNED(struct uncached_list, rt_uncached_list); 1520 1521 void rt_add_uncached_list(struct rtable *rt) 1522 { 1523 struct uncached_list *ul = raw_cpu_ptr(&rt_uncached_list); 1524 1525 rt->rt_uncached_list = ul; 1526 1527 spin_lock_bh(&ul->lock); 1528 list_add_tail(&rt->rt_uncached, &ul->head); 1529 spin_unlock_bh(&ul->lock); 1530 } 1531 1532 void rt_del_uncached_list(struct rtable *rt) 1533 { 1534 if (!list_empty(&rt->rt_uncached)) { 1535 struct uncached_list *ul = rt->rt_uncached_list; 1536 1537 spin_lock_bh(&ul->lock); 1538 list_del(&rt->rt_uncached); 1539 spin_unlock_bh(&ul->lock); 1540 } 1541 } 1542 1543 static void ipv4_dst_destroy(struct dst_entry *dst) 1544 { 1545 struct rtable *rt = (struct rtable *)dst; 1546 1547 ip_dst_metrics_put(dst); 1548 rt_del_uncached_list(rt); 1549 } 1550 1551 void rt_flush_dev(struct net_device *dev) 1552 { 1553 struct rtable *rt; 1554 int cpu; 1555 1556 for_each_possible_cpu(cpu) { 1557 struct uncached_list *ul = &per_cpu(rt_uncached_list, cpu); 1558 1559 spin_lock_bh(&ul->lock); 1560 list_for_each_entry(rt, &ul->head, rt_uncached) { 1561 if (rt->dst.dev != dev) 1562 continue; 1563 rt->dst.dev = blackhole_netdev; 1564 dev_hold(rt->dst.dev); 1565 dev_put(dev); 1566 } 1567 spin_unlock_bh(&ul->lock); 1568 } 1569 } 1570 1571 static bool rt_cache_valid(const struct rtable *rt) 1572 { 1573 return rt && 1574 rt->dst.obsolete == DST_OBSOLETE_FORCE_CHK && 1575 !rt_is_expired(rt); 1576 } 1577 1578 static void rt_set_nexthop(struct rtable *rt, __be32 daddr, 1579 const struct fib_result *res, 1580 struct fib_nh_exception *fnhe, 1581 struct fib_info *fi, u16 type, u32 itag, 1582 const bool do_cache) 1583 { 1584 bool cached = false; 1585 1586 if (fi) { 1587 struct fib_nh_common *nhc = FIB_RES_NHC(*res); 1588 1589 if (nhc->nhc_gw_family && nhc->nhc_scope == RT_SCOPE_LINK) { 1590 rt->rt_uses_gateway = 1; 1591 rt->rt_gw_family = nhc->nhc_gw_family; 1592 /* only INET and INET6 are supported */ 1593 if (likely(nhc->nhc_gw_family == AF_INET)) 1594 rt->rt_gw4 = nhc->nhc_gw.ipv4; 1595 else 1596 rt->rt_gw6 = nhc->nhc_gw.ipv6; 1597 } 1598 1599 ip_dst_init_metrics(&rt->dst, fi->fib_metrics); 1600 1601 #ifdef CONFIG_IP_ROUTE_CLASSID 1602 if (nhc->nhc_family == AF_INET) { 1603 struct fib_nh *nh; 1604 1605 nh = container_of(nhc, struct fib_nh, nh_common); 1606 rt->dst.tclassid = nh->nh_tclassid; 1607 } 1608 #endif 1609 rt->dst.lwtstate = lwtstate_get(nhc->nhc_lwtstate); 1610 if (unlikely(fnhe)) 1611 cached = rt_bind_exception(rt, fnhe, daddr, do_cache); 1612 else if (do_cache) 1613 cached = rt_cache_route(nhc, rt); 1614 if (unlikely(!cached)) { 1615 /* Routes we intend to cache in nexthop exception or 1616 * FIB nexthop have the DST_NOCACHE bit clear. 1617 * However, if we are unsuccessful at storing this 1618 * route into the cache we really need to set it. 1619 */ 1620 if (!rt->rt_gw4) { 1621 rt->rt_gw_family = AF_INET; 1622 rt->rt_gw4 = daddr; 1623 } 1624 rt_add_uncached_list(rt); 1625 } 1626 } else 1627 rt_add_uncached_list(rt); 1628 1629 #ifdef CONFIG_IP_ROUTE_CLASSID 1630 #ifdef CONFIG_IP_MULTIPLE_TABLES 1631 set_class_tag(rt, res->tclassid); 1632 #endif 1633 set_class_tag(rt, itag); 1634 #endif 1635 } 1636 1637 struct rtable *rt_dst_alloc(struct net_device *dev, 1638 unsigned int flags, u16 type, 1639 bool nopolicy, bool noxfrm) 1640 { 1641 struct rtable *rt; 1642 1643 rt = dst_alloc(&ipv4_dst_ops, dev, 1, DST_OBSOLETE_FORCE_CHK, 1644 (nopolicy ? DST_NOPOLICY : 0) | 1645 (noxfrm ? DST_NOXFRM : 0)); 1646 1647 if (rt) { 1648 rt->rt_genid = rt_genid_ipv4(dev_net(dev)); 1649 rt->rt_flags = flags; 1650 rt->rt_type = type; 1651 rt->rt_is_input = 0; 1652 rt->rt_iif = 0; 1653 rt->rt_pmtu = 0; 1654 rt->rt_mtu_locked = 0; 1655 rt->rt_uses_gateway = 0; 1656 rt->rt_gw_family = 0; 1657 rt->rt_gw4 = 0; 1658 INIT_LIST_HEAD(&rt->rt_uncached); 1659 1660 rt->dst.output = ip_output; 1661 if (flags & RTCF_LOCAL) 1662 rt->dst.input = ip_local_deliver; 1663 } 1664 1665 return rt; 1666 } 1667 EXPORT_SYMBOL(rt_dst_alloc); 1668 1669 struct rtable *rt_dst_clone(struct net_device *dev, struct rtable *rt) 1670 { 1671 struct rtable *new_rt; 1672 1673 new_rt = dst_alloc(&ipv4_dst_ops, dev, 1, DST_OBSOLETE_FORCE_CHK, 1674 rt->dst.flags); 1675 1676 if (new_rt) { 1677 new_rt->rt_genid = rt_genid_ipv4(dev_net(dev)); 1678 new_rt->rt_flags = rt->rt_flags; 1679 new_rt->rt_type = rt->rt_type; 1680 new_rt->rt_is_input = rt->rt_is_input; 1681 new_rt->rt_iif = rt->rt_iif; 1682 new_rt->rt_pmtu = rt->rt_pmtu; 1683 new_rt->rt_mtu_locked = rt->rt_mtu_locked; 1684 new_rt->rt_gw_family = rt->rt_gw_family; 1685 if (rt->rt_gw_family == AF_INET) 1686 new_rt->rt_gw4 = rt->rt_gw4; 1687 else if (rt->rt_gw_family == AF_INET6) 1688 new_rt->rt_gw6 = rt->rt_gw6; 1689 INIT_LIST_HEAD(&new_rt->rt_uncached); 1690 1691 new_rt->dst.input = rt->dst.input; 1692 new_rt->dst.output = rt->dst.output; 1693 new_rt->dst.error = rt->dst.error; 1694 new_rt->dst.lastuse = jiffies; 1695 new_rt->dst.lwtstate = lwtstate_get(rt->dst.lwtstate); 1696 } 1697 return new_rt; 1698 } 1699 EXPORT_SYMBOL(rt_dst_clone); 1700 1701 /* called in rcu_read_lock() section */ 1702 int ip_mc_validate_source(struct sk_buff *skb, __be32 daddr, __be32 saddr, 1703 u8 tos, struct net_device *dev, 1704 struct in_device *in_dev, u32 *itag) 1705 { 1706 int err; 1707 1708 /* Primary sanity checks. */ 1709 if (!in_dev) 1710 return -EINVAL; 1711 1712 if (ipv4_is_multicast(saddr) || ipv4_is_lbcast(saddr) || 1713 skb->protocol != htons(ETH_P_IP)) 1714 return -EINVAL; 1715 1716 if (ipv4_is_loopback(saddr) && !IN_DEV_ROUTE_LOCALNET(in_dev)) 1717 return -EINVAL; 1718 1719 if (ipv4_is_zeronet(saddr)) { 1720 if (!ipv4_is_local_multicast(daddr) && 1721 ip_hdr(skb)->protocol != IPPROTO_IGMP) 1722 return -EINVAL; 1723 } else { 1724 err = fib_validate_source(skb, saddr, 0, tos, 0, dev, 1725 in_dev, itag); 1726 if (err < 0) 1727 return err; 1728 } 1729 return 0; 1730 } 1731 1732 /* called in rcu_read_lock() section */ 1733 static int ip_route_input_mc(struct sk_buff *skb, __be32 daddr, __be32 saddr, 1734 u8 tos, struct net_device *dev, int our) 1735 { 1736 struct in_device *in_dev = __in_dev_get_rcu(dev); 1737 unsigned int flags = RTCF_MULTICAST; 1738 struct rtable *rth; 1739 u32 itag = 0; 1740 int err; 1741 1742 err = ip_mc_validate_source(skb, daddr, saddr, tos, dev, in_dev, &itag); 1743 if (err) 1744 return err; 1745 1746 if (our) 1747 flags |= RTCF_LOCAL; 1748 1749 rth = rt_dst_alloc(dev_net(dev)->loopback_dev, flags, RTN_MULTICAST, 1750 IN_DEV_ORCONF(in_dev, NOPOLICY), false); 1751 if (!rth) 1752 return -ENOBUFS; 1753 1754 #ifdef CONFIG_IP_ROUTE_CLASSID 1755 rth->dst.tclassid = itag; 1756 #endif 1757 rth->dst.output = ip_rt_bug; 1758 rth->rt_is_input= 1; 1759 1760 #ifdef CONFIG_IP_MROUTE 1761 if (!ipv4_is_local_multicast(daddr) && IN_DEV_MFORWARD(in_dev)) 1762 rth->dst.input = ip_mr_input; 1763 #endif 1764 RT_CACHE_STAT_INC(in_slow_mc); 1765 1766 skb_dst_set(skb, &rth->dst); 1767 return 0; 1768 } 1769 1770 1771 static void ip_handle_martian_source(struct net_device *dev, 1772 struct in_device *in_dev, 1773 struct sk_buff *skb, 1774 __be32 daddr, 1775 __be32 saddr) 1776 { 1777 RT_CACHE_STAT_INC(in_martian_src); 1778 #ifdef CONFIG_IP_ROUTE_VERBOSE 1779 if (IN_DEV_LOG_MARTIANS(in_dev) && net_ratelimit()) { 1780 /* 1781 * RFC1812 recommendation, if source is martian, 1782 * the only hint is MAC header. 1783 */ 1784 pr_warn("martian source %pI4 from %pI4, on dev %s\n", 1785 &daddr, &saddr, dev->name); 1786 if (dev->hard_header_len && skb_mac_header_was_set(skb)) { 1787 print_hex_dump(KERN_WARNING, "ll header: ", 1788 DUMP_PREFIX_OFFSET, 16, 1, 1789 skb_mac_header(skb), 1790 dev->hard_header_len, false); 1791 } 1792 } 1793 #endif 1794 } 1795 1796 /* called in rcu_read_lock() section */ 1797 static int __mkroute_input(struct sk_buff *skb, 1798 const struct fib_result *res, 1799 struct in_device *in_dev, 1800 __be32 daddr, __be32 saddr, u32 tos) 1801 { 1802 struct fib_nh_common *nhc = FIB_RES_NHC(*res); 1803 struct net_device *dev = nhc->nhc_dev; 1804 struct fib_nh_exception *fnhe; 1805 struct rtable *rth; 1806 int err; 1807 struct in_device *out_dev; 1808 bool do_cache; 1809 u32 itag = 0; 1810 1811 /* get a working reference to the output device */ 1812 out_dev = __in_dev_get_rcu(dev); 1813 if (!out_dev) { 1814 net_crit_ratelimited("Bug in ip_route_input_slow(). Please report.\n"); 1815 return -EINVAL; 1816 } 1817 1818 err = fib_validate_source(skb, saddr, daddr, tos, FIB_RES_OIF(*res), 1819 in_dev->dev, in_dev, &itag); 1820 if (err < 0) { 1821 ip_handle_martian_source(in_dev->dev, in_dev, skb, daddr, 1822 saddr); 1823 1824 goto cleanup; 1825 } 1826 1827 do_cache = res->fi && !itag; 1828 if (out_dev == in_dev && err && IN_DEV_TX_REDIRECTS(out_dev) && 1829 skb->protocol == htons(ETH_P_IP)) { 1830 __be32 gw; 1831 1832 gw = nhc->nhc_gw_family == AF_INET ? nhc->nhc_gw.ipv4 : 0; 1833 if (IN_DEV_SHARED_MEDIA(out_dev) || 1834 inet_addr_onlink(out_dev, saddr, gw)) 1835 IPCB(skb)->flags |= IPSKB_DOREDIRECT; 1836 } 1837 1838 if (skb->protocol != htons(ETH_P_IP)) { 1839 /* Not IP (i.e. ARP). Do not create route, if it is 1840 * invalid for proxy arp. DNAT routes are always valid. 1841 * 1842 * Proxy arp feature have been extended to allow, ARP 1843 * replies back to the same interface, to support 1844 * Private VLAN switch technologies. See arp.c. 1845 */ 1846 if (out_dev == in_dev && 1847 IN_DEV_PROXY_ARP_PVLAN(in_dev) == 0) { 1848 err = -EINVAL; 1849 goto cleanup; 1850 } 1851 } 1852 1853 fnhe = find_exception(nhc, daddr); 1854 if (do_cache) { 1855 if (fnhe) 1856 rth = rcu_dereference(fnhe->fnhe_rth_input); 1857 else 1858 rth = rcu_dereference(nhc->nhc_rth_input); 1859 if (rt_cache_valid(rth)) { 1860 skb_dst_set_noref(skb, &rth->dst); 1861 goto out; 1862 } 1863 } 1864 1865 rth = rt_dst_alloc(out_dev->dev, 0, res->type, 1866 IN_DEV_ORCONF(in_dev, NOPOLICY), 1867 IN_DEV_ORCONF(out_dev, NOXFRM)); 1868 if (!rth) { 1869 err = -ENOBUFS; 1870 goto cleanup; 1871 } 1872 1873 rth->rt_is_input = 1; 1874 RT_CACHE_STAT_INC(in_slow_tot); 1875 1876 rth->dst.input = ip_forward; 1877 1878 rt_set_nexthop(rth, daddr, res, fnhe, res->fi, res->type, itag, 1879 do_cache); 1880 lwtunnel_set_redirect(&rth->dst); 1881 skb_dst_set(skb, &rth->dst); 1882 out: 1883 err = 0; 1884 cleanup: 1885 return err; 1886 } 1887 1888 #ifdef CONFIG_IP_ROUTE_MULTIPATH 1889 /* To make ICMP packets follow the right flow, the multipath hash is 1890 * calculated from the inner IP addresses. 1891 */ 1892 static void ip_multipath_l3_keys(const struct sk_buff *skb, 1893 struct flow_keys *hash_keys) 1894 { 1895 const struct iphdr *outer_iph = ip_hdr(skb); 1896 const struct iphdr *key_iph = outer_iph; 1897 const struct iphdr *inner_iph; 1898 const struct icmphdr *icmph; 1899 struct iphdr _inner_iph; 1900 struct icmphdr _icmph; 1901 1902 if (likely(outer_iph->protocol != IPPROTO_ICMP)) 1903 goto out; 1904 1905 if (unlikely((outer_iph->frag_off & htons(IP_OFFSET)) != 0)) 1906 goto out; 1907 1908 icmph = skb_header_pointer(skb, outer_iph->ihl * 4, sizeof(_icmph), 1909 &_icmph); 1910 if (!icmph) 1911 goto out; 1912 1913 if (!icmp_is_err(icmph->type)) 1914 goto out; 1915 1916 inner_iph = skb_header_pointer(skb, 1917 outer_iph->ihl * 4 + sizeof(_icmph), 1918 sizeof(_inner_iph), &_inner_iph); 1919 if (!inner_iph) 1920 goto out; 1921 1922 key_iph = inner_iph; 1923 out: 1924 hash_keys->addrs.v4addrs.src = key_iph->saddr; 1925 hash_keys->addrs.v4addrs.dst = key_iph->daddr; 1926 } 1927 1928 /* if skb is set it will be used and fl4 can be NULL */ 1929 int fib_multipath_hash(const struct net *net, const struct flowi4 *fl4, 1930 const struct sk_buff *skb, struct flow_keys *flkeys) 1931 { 1932 u32 multipath_hash = fl4 ? fl4->flowi4_multipath_hash : 0; 1933 struct flow_keys hash_keys; 1934 u32 mhash; 1935 1936 switch (net->ipv4.sysctl_fib_multipath_hash_policy) { 1937 case 0: 1938 memset(&hash_keys, 0, sizeof(hash_keys)); 1939 hash_keys.control.addr_type = FLOW_DISSECTOR_KEY_IPV4_ADDRS; 1940 if (skb) { 1941 ip_multipath_l3_keys(skb, &hash_keys); 1942 } else { 1943 hash_keys.addrs.v4addrs.src = fl4->saddr; 1944 hash_keys.addrs.v4addrs.dst = fl4->daddr; 1945 } 1946 break; 1947 case 1: 1948 /* skb is currently provided only when forwarding */ 1949 if (skb) { 1950 unsigned int flag = FLOW_DISSECTOR_F_STOP_AT_ENCAP; 1951 struct flow_keys keys; 1952 1953 /* short-circuit if we already have L4 hash present */ 1954 if (skb->l4_hash) 1955 return skb_get_hash_raw(skb) >> 1; 1956 1957 memset(&hash_keys, 0, sizeof(hash_keys)); 1958 1959 if (!flkeys) { 1960 skb_flow_dissect_flow_keys(skb, &keys, flag); 1961 flkeys = &keys; 1962 } 1963 1964 hash_keys.control.addr_type = FLOW_DISSECTOR_KEY_IPV4_ADDRS; 1965 hash_keys.addrs.v4addrs.src = flkeys->addrs.v4addrs.src; 1966 hash_keys.addrs.v4addrs.dst = flkeys->addrs.v4addrs.dst; 1967 hash_keys.ports.src = flkeys->ports.src; 1968 hash_keys.ports.dst = flkeys->ports.dst; 1969 hash_keys.basic.ip_proto = flkeys->basic.ip_proto; 1970 } else { 1971 memset(&hash_keys, 0, sizeof(hash_keys)); 1972 hash_keys.control.addr_type = FLOW_DISSECTOR_KEY_IPV4_ADDRS; 1973 hash_keys.addrs.v4addrs.src = fl4->saddr; 1974 hash_keys.addrs.v4addrs.dst = fl4->daddr; 1975 hash_keys.ports.src = fl4->fl4_sport; 1976 hash_keys.ports.dst = fl4->fl4_dport; 1977 hash_keys.basic.ip_proto = fl4->flowi4_proto; 1978 } 1979 break; 1980 case 2: 1981 memset(&hash_keys, 0, sizeof(hash_keys)); 1982 /* skb is currently provided only when forwarding */ 1983 if (skb) { 1984 struct flow_keys keys; 1985 1986 skb_flow_dissect_flow_keys(skb, &keys, 0); 1987 /* Inner can be v4 or v6 */ 1988 if (keys.control.addr_type == FLOW_DISSECTOR_KEY_IPV4_ADDRS) { 1989 hash_keys.control.addr_type = FLOW_DISSECTOR_KEY_IPV4_ADDRS; 1990 hash_keys.addrs.v4addrs.src = keys.addrs.v4addrs.src; 1991 hash_keys.addrs.v4addrs.dst = keys.addrs.v4addrs.dst; 1992 } else if (keys.control.addr_type == FLOW_DISSECTOR_KEY_IPV6_ADDRS) { 1993 hash_keys.control.addr_type = FLOW_DISSECTOR_KEY_IPV6_ADDRS; 1994 hash_keys.addrs.v6addrs.src = keys.addrs.v6addrs.src; 1995 hash_keys.addrs.v6addrs.dst = keys.addrs.v6addrs.dst; 1996 hash_keys.tags.flow_label = keys.tags.flow_label; 1997 hash_keys.basic.ip_proto = keys.basic.ip_proto; 1998 } else { 1999 /* Same as case 0 */ 2000 hash_keys.control.addr_type = FLOW_DISSECTOR_KEY_IPV4_ADDRS; 2001 ip_multipath_l3_keys(skb, &hash_keys); 2002 } 2003 } else { 2004 /* Same as case 0 */ 2005 hash_keys.control.addr_type = FLOW_DISSECTOR_KEY_IPV4_ADDRS; 2006 hash_keys.addrs.v4addrs.src = fl4->saddr; 2007 hash_keys.addrs.v4addrs.dst = fl4->daddr; 2008 } 2009 break; 2010 } 2011 mhash = flow_hash_from_keys(&hash_keys); 2012 2013 if (multipath_hash) 2014 mhash = jhash_2words(mhash, multipath_hash, 0); 2015 2016 return mhash >> 1; 2017 } 2018 #endif /* CONFIG_IP_ROUTE_MULTIPATH */ 2019 2020 static int ip_mkroute_input(struct sk_buff *skb, 2021 struct fib_result *res, 2022 struct in_device *in_dev, 2023 __be32 daddr, __be32 saddr, u32 tos, 2024 struct flow_keys *hkeys) 2025 { 2026 #ifdef CONFIG_IP_ROUTE_MULTIPATH 2027 if (res->fi && fib_info_num_path(res->fi) > 1) { 2028 int h = fib_multipath_hash(res->fi->fib_net, NULL, skb, hkeys); 2029 2030 fib_select_multipath(res, h); 2031 } 2032 #endif 2033 2034 /* create a routing cache entry */ 2035 return __mkroute_input(skb, res, in_dev, daddr, saddr, tos); 2036 } 2037 2038 /* Implements all the saddr-related checks as ip_route_input_slow(), 2039 * assuming daddr is valid and the destination is not a local broadcast one. 2040 * Uses the provided hint instead of performing a route lookup. 2041 */ 2042 int ip_route_use_hint(struct sk_buff *skb, __be32 daddr, __be32 saddr, 2043 u8 tos, struct net_device *dev, 2044 const struct sk_buff *hint) 2045 { 2046 struct in_device *in_dev = __in_dev_get_rcu(dev); 2047 struct rtable *rt = skb_rtable(hint); 2048 struct net *net = dev_net(dev); 2049 int err = -EINVAL; 2050 u32 tag = 0; 2051 2052 if (ipv4_is_multicast(saddr) || ipv4_is_lbcast(saddr)) 2053 goto martian_source; 2054 2055 if (ipv4_is_zeronet(saddr)) 2056 goto martian_source; 2057 2058 if (ipv4_is_loopback(saddr) && !IN_DEV_NET_ROUTE_LOCALNET(in_dev, net)) 2059 goto martian_source; 2060 2061 if (rt->rt_type != RTN_LOCAL) 2062 goto skip_validate_source; 2063 2064 tos &= IPTOS_RT_MASK; 2065 err = fib_validate_source(skb, saddr, daddr, tos, 0, dev, in_dev, &tag); 2066 if (err < 0) 2067 goto martian_source; 2068 2069 skip_validate_source: 2070 skb_dst_copy(skb, hint); 2071 return 0; 2072 2073 martian_source: 2074 ip_handle_martian_source(dev, in_dev, skb, daddr, saddr); 2075 return err; 2076 } 2077 2078 /* 2079 * NOTE. We drop all the packets that has local source 2080 * addresses, because every properly looped back packet 2081 * must have correct destination already attached by output routine. 2082 * Changes in the enforced policies must be applied also to 2083 * ip_route_use_hint(). 2084 * 2085 * Such approach solves two big problems: 2086 * 1. Not simplex devices are handled properly. 2087 * 2. IP spoofing attempts are filtered with 100% of guarantee. 2088 * called with rcu_read_lock() 2089 */ 2090 2091 static int ip_route_input_slow(struct sk_buff *skb, __be32 daddr, __be32 saddr, 2092 u8 tos, struct net_device *dev, 2093 struct fib_result *res) 2094 { 2095 struct in_device *in_dev = __in_dev_get_rcu(dev); 2096 struct flow_keys *flkeys = NULL, _flkeys; 2097 struct net *net = dev_net(dev); 2098 struct ip_tunnel_info *tun_info; 2099 int err = -EINVAL; 2100 unsigned int flags = 0; 2101 u32 itag = 0; 2102 struct rtable *rth; 2103 struct flowi4 fl4; 2104 bool do_cache = true; 2105 2106 /* IP on this device is disabled. */ 2107 2108 if (!in_dev) 2109 goto out; 2110 2111 /* Check for the most weird martians, which can be not detected 2112 * by fib_lookup. 2113 */ 2114 2115 tun_info = skb_tunnel_info(skb); 2116 if (tun_info && !(tun_info->mode & IP_TUNNEL_INFO_TX)) 2117 fl4.flowi4_tun_key.tun_id = tun_info->key.tun_id; 2118 else 2119 fl4.flowi4_tun_key.tun_id = 0; 2120 skb_dst_drop(skb); 2121 2122 if (ipv4_is_multicast(saddr) || ipv4_is_lbcast(saddr)) 2123 goto martian_source; 2124 2125 res->fi = NULL; 2126 res->table = NULL; 2127 if (ipv4_is_lbcast(daddr) || (saddr == 0 && daddr == 0)) 2128 goto brd_input; 2129 2130 /* Accept zero addresses only to limited broadcast; 2131 * I even do not know to fix it or not. Waiting for complains :-) 2132 */ 2133 if (ipv4_is_zeronet(saddr)) 2134 goto martian_source; 2135 2136 if (ipv4_is_zeronet(daddr)) 2137 goto martian_destination; 2138 2139 /* Following code try to avoid calling IN_DEV_NET_ROUTE_LOCALNET(), 2140 * and call it once if daddr or/and saddr are loopback addresses 2141 */ 2142 if (ipv4_is_loopback(daddr)) { 2143 if (!IN_DEV_NET_ROUTE_LOCALNET(in_dev, net)) 2144 goto martian_destination; 2145 } else if (ipv4_is_loopback(saddr)) { 2146 if (!IN_DEV_NET_ROUTE_LOCALNET(in_dev, net)) 2147 goto martian_source; 2148 } 2149 2150 /* 2151 * Now we are ready to route packet. 2152 */ 2153 fl4.flowi4_oif = 0; 2154 fl4.flowi4_iif = dev->ifindex; 2155 fl4.flowi4_mark = skb->mark; 2156 fl4.flowi4_tos = tos; 2157 fl4.flowi4_scope = RT_SCOPE_UNIVERSE; 2158 fl4.flowi4_flags = 0; 2159 fl4.daddr = daddr; 2160 fl4.saddr = saddr; 2161 fl4.flowi4_uid = sock_net_uid(net, NULL); 2162 fl4.flowi4_multipath_hash = 0; 2163 2164 if (fib4_rules_early_flow_dissect(net, skb, &fl4, &_flkeys)) { 2165 flkeys = &_flkeys; 2166 } else { 2167 fl4.flowi4_proto = 0; 2168 fl4.fl4_sport = 0; 2169 fl4.fl4_dport = 0; 2170 } 2171 2172 err = fib_lookup(net, &fl4, res, 0); 2173 if (err != 0) { 2174 if (!IN_DEV_FORWARD(in_dev)) 2175 err = -EHOSTUNREACH; 2176 goto no_route; 2177 } 2178 2179 if (res->type == RTN_BROADCAST) { 2180 if (IN_DEV_BFORWARD(in_dev)) 2181 goto make_route; 2182 /* not do cache if bc_forwarding is enabled */ 2183 if (IPV4_DEVCONF_ALL(net, BC_FORWARDING)) 2184 do_cache = false; 2185 goto brd_input; 2186 } 2187 2188 if (res->type == RTN_LOCAL) { 2189 err = fib_validate_source(skb, saddr, daddr, tos, 2190 0, dev, in_dev, &itag); 2191 if (err < 0) 2192 goto martian_source; 2193 goto local_input; 2194 } 2195 2196 if (!IN_DEV_FORWARD(in_dev)) { 2197 err = -EHOSTUNREACH; 2198 goto no_route; 2199 } 2200 if (res->type != RTN_UNICAST) 2201 goto martian_destination; 2202 2203 make_route: 2204 err = ip_mkroute_input(skb, res, in_dev, daddr, saddr, tos, flkeys); 2205 out: return err; 2206 2207 brd_input: 2208 if (skb->protocol != htons(ETH_P_IP)) 2209 goto e_inval; 2210 2211 if (!ipv4_is_zeronet(saddr)) { 2212 err = fib_validate_source(skb, saddr, 0, tos, 0, dev, 2213 in_dev, &itag); 2214 if (err < 0) 2215 goto martian_source; 2216 } 2217 flags |= RTCF_BROADCAST; 2218 res->type = RTN_BROADCAST; 2219 RT_CACHE_STAT_INC(in_brd); 2220 2221 local_input: 2222 do_cache &= res->fi && !itag; 2223 if (do_cache) { 2224 struct fib_nh_common *nhc = FIB_RES_NHC(*res); 2225 2226 rth = rcu_dereference(nhc->nhc_rth_input); 2227 if (rt_cache_valid(rth)) { 2228 skb_dst_set_noref(skb, &rth->dst); 2229 err = 0; 2230 goto out; 2231 } 2232 } 2233 2234 rth = rt_dst_alloc(l3mdev_master_dev_rcu(dev) ? : net->loopback_dev, 2235 flags | RTCF_LOCAL, res->type, 2236 IN_DEV_ORCONF(in_dev, NOPOLICY), false); 2237 if (!rth) 2238 goto e_nobufs; 2239 2240 rth->dst.output= ip_rt_bug; 2241 #ifdef CONFIG_IP_ROUTE_CLASSID 2242 rth->dst.tclassid = itag; 2243 #endif 2244 rth->rt_is_input = 1; 2245 2246 RT_CACHE_STAT_INC(in_slow_tot); 2247 if (res->type == RTN_UNREACHABLE) { 2248 rth->dst.input= ip_error; 2249 rth->dst.error= -err; 2250 rth->rt_flags &= ~RTCF_LOCAL; 2251 } 2252 2253 if (do_cache) { 2254 struct fib_nh_common *nhc = FIB_RES_NHC(*res); 2255 2256 rth->dst.lwtstate = lwtstate_get(nhc->nhc_lwtstate); 2257 if (lwtunnel_input_redirect(rth->dst.lwtstate)) { 2258 WARN_ON(rth->dst.input == lwtunnel_input); 2259 rth->dst.lwtstate->orig_input = rth->dst.input; 2260 rth->dst.input = lwtunnel_input; 2261 } 2262 2263 if (unlikely(!rt_cache_route(nhc, rth))) 2264 rt_add_uncached_list(rth); 2265 } 2266 skb_dst_set(skb, &rth->dst); 2267 err = 0; 2268 goto out; 2269 2270 no_route: 2271 RT_CACHE_STAT_INC(in_no_route); 2272 res->type = RTN_UNREACHABLE; 2273 res->fi = NULL; 2274 res->table = NULL; 2275 goto local_input; 2276 2277 /* 2278 * Do not cache martian addresses: they should be logged (RFC1812) 2279 */ 2280 martian_destination: 2281 RT_CACHE_STAT_INC(in_martian_dst); 2282 #ifdef CONFIG_IP_ROUTE_VERBOSE 2283 if (IN_DEV_LOG_MARTIANS(in_dev)) 2284 net_warn_ratelimited("martian destination %pI4 from %pI4, dev %s\n", 2285 &daddr, &saddr, dev->name); 2286 #endif 2287 2288 e_inval: 2289 err = -EINVAL; 2290 goto out; 2291 2292 e_nobufs: 2293 err = -ENOBUFS; 2294 goto out; 2295 2296 martian_source: 2297 ip_handle_martian_source(dev, in_dev, skb, daddr, saddr); 2298 goto out; 2299 } 2300 2301 int ip_route_input_noref(struct sk_buff *skb, __be32 daddr, __be32 saddr, 2302 u8 tos, struct net_device *dev) 2303 { 2304 struct fib_result res; 2305 int err; 2306 2307 tos &= IPTOS_RT_MASK; 2308 rcu_read_lock(); 2309 err = ip_route_input_rcu(skb, daddr, saddr, tos, dev, &res); 2310 rcu_read_unlock(); 2311 2312 return err; 2313 } 2314 EXPORT_SYMBOL(ip_route_input_noref); 2315 2316 /* called with rcu_read_lock held */ 2317 int ip_route_input_rcu(struct sk_buff *skb, __be32 daddr, __be32 saddr, 2318 u8 tos, struct net_device *dev, struct fib_result *res) 2319 { 2320 /* Multicast recognition logic is moved from route cache to here. 2321 * The problem was that too many Ethernet cards have broken/missing 2322 * hardware multicast filters :-( As result the host on multicasting 2323 * network acquires a lot of useless route cache entries, sort of 2324 * SDR messages from all the world. Now we try to get rid of them. 2325 * Really, provided software IP multicast filter is organized 2326 * reasonably (at least, hashed), it does not result in a slowdown 2327 * comparing with route cache reject entries. 2328 * Note, that multicast routers are not affected, because 2329 * route cache entry is created eventually. 2330 */ 2331 if (ipv4_is_multicast(daddr)) { 2332 struct in_device *in_dev = __in_dev_get_rcu(dev); 2333 int our = 0; 2334 int err = -EINVAL; 2335 2336 if (!in_dev) 2337 return err; 2338 our = ip_check_mc_rcu(in_dev, daddr, saddr, 2339 ip_hdr(skb)->protocol); 2340 2341 /* check l3 master if no match yet */ 2342 if (!our && netif_is_l3_slave(dev)) { 2343 struct in_device *l3_in_dev; 2344 2345 l3_in_dev = __in_dev_get_rcu(skb->dev); 2346 if (l3_in_dev) 2347 our = ip_check_mc_rcu(l3_in_dev, daddr, saddr, 2348 ip_hdr(skb)->protocol); 2349 } 2350 2351 if (our 2352 #ifdef CONFIG_IP_MROUTE 2353 || 2354 (!ipv4_is_local_multicast(daddr) && 2355 IN_DEV_MFORWARD(in_dev)) 2356 #endif 2357 ) { 2358 err = ip_route_input_mc(skb, daddr, saddr, 2359 tos, dev, our); 2360 } 2361 return err; 2362 } 2363 2364 return ip_route_input_slow(skb, daddr, saddr, tos, dev, res); 2365 } 2366 2367 /* called with rcu_read_lock() */ 2368 static struct rtable *__mkroute_output(const struct fib_result *res, 2369 const struct flowi4 *fl4, int orig_oif, 2370 struct net_device *dev_out, 2371 unsigned int flags) 2372 { 2373 struct fib_info *fi = res->fi; 2374 struct fib_nh_exception *fnhe; 2375 struct in_device *in_dev; 2376 u16 type = res->type; 2377 struct rtable *rth; 2378 bool do_cache; 2379 2380 in_dev = __in_dev_get_rcu(dev_out); 2381 if (!in_dev) 2382 return ERR_PTR(-EINVAL); 2383 2384 if (likely(!IN_DEV_ROUTE_LOCALNET(in_dev))) 2385 if (ipv4_is_loopback(fl4->saddr) && 2386 !(dev_out->flags & IFF_LOOPBACK) && 2387 !netif_is_l3_master(dev_out)) 2388 return ERR_PTR(-EINVAL); 2389 2390 if (ipv4_is_lbcast(fl4->daddr)) 2391 type = RTN_BROADCAST; 2392 else if (ipv4_is_multicast(fl4->daddr)) 2393 type = RTN_MULTICAST; 2394 else if (ipv4_is_zeronet(fl4->daddr)) 2395 return ERR_PTR(-EINVAL); 2396 2397 if (dev_out->flags & IFF_LOOPBACK) 2398 flags |= RTCF_LOCAL; 2399 2400 do_cache = true; 2401 if (type == RTN_BROADCAST) { 2402 flags |= RTCF_BROADCAST | RTCF_LOCAL; 2403 fi = NULL; 2404 } else if (type == RTN_MULTICAST) { 2405 flags |= RTCF_MULTICAST | RTCF_LOCAL; 2406 if (!ip_check_mc_rcu(in_dev, fl4->daddr, fl4->saddr, 2407 fl4->flowi4_proto)) 2408 flags &= ~RTCF_LOCAL; 2409 else 2410 do_cache = false; 2411 /* If multicast route do not exist use 2412 * default one, but do not gateway in this case. 2413 * Yes, it is hack. 2414 */ 2415 if (fi && res->prefixlen < 4) 2416 fi = NULL; 2417 } else if ((type == RTN_LOCAL) && (orig_oif != 0) && 2418 (orig_oif != dev_out->ifindex)) { 2419 /* For local routes that require a particular output interface 2420 * we do not want to cache the result. Caching the result 2421 * causes incorrect behaviour when there are multiple source 2422 * addresses on the interface, the end result being that if the 2423 * intended recipient is waiting on that interface for the 2424 * packet he won't receive it because it will be delivered on 2425 * the loopback interface and the IP_PKTINFO ipi_ifindex will 2426 * be set to the loopback interface as well. 2427 */ 2428 do_cache = false; 2429 } 2430 2431 fnhe = NULL; 2432 do_cache &= fi != NULL; 2433 if (fi) { 2434 struct fib_nh_common *nhc = FIB_RES_NHC(*res); 2435 struct rtable __rcu **prth; 2436 2437 fnhe = find_exception(nhc, fl4->daddr); 2438 if (!do_cache) 2439 goto add; 2440 if (fnhe) { 2441 prth = &fnhe->fnhe_rth_output; 2442 } else { 2443 if (unlikely(fl4->flowi4_flags & 2444 FLOWI_FLAG_KNOWN_NH && 2445 !(nhc->nhc_gw_family && 2446 nhc->nhc_scope == RT_SCOPE_LINK))) { 2447 do_cache = false; 2448 goto add; 2449 } 2450 prth = raw_cpu_ptr(nhc->nhc_pcpu_rth_output); 2451 } 2452 rth = rcu_dereference(*prth); 2453 if (rt_cache_valid(rth) && dst_hold_safe(&rth->dst)) 2454 return rth; 2455 } 2456 2457 add: 2458 rth = rt_dst_alloc(dev_out, flags, type, 2459 IN_DEV_ORCONF(in_dev, NOPOLICY), 2460 IN_DEV_ORCONF(in_dev, NOXFRM)); 2461 if (!rth) 2462 return ERR_PTR(-ENOBUFS); 2463 2464 rth->rt_iif = orig_oif; 2465 2466 RT_CACHE_STAT_INC(out_slow_tot); 2467 2468 if (flags & (RTCF_BROADCAST | RTCF_MULTICAST)) { 2469 if (flags & RTCF_LOCAL && 2470 !(dev_out->flags & IFF_LOOPBACK)) { 2471 rth->dst.output = ip_mc_output; 2472 RT_CACHE_STAT_INC(out_slow_mc); 2473 } 2474 #ifdef CONFIG_IP_MROUTE 2475 if (type == RTN_MULTICAST) { 2476 if (IN_DEV_MFORWARD(in_dev) && 2477 !ipv4_is_local_multicast(fl4->daddr)) { 2478 rth->dst.input = ip_mr_input; 2479 rth->dst.output = ip_mc_output; 2480 } 2481 } 2482 #endif 2483 } 2484 2485 rt_set_nexthop(rth, fl4->daddr, res, fnhe, fi, type, 0, do_cache); 2486 lwtunnel_set_redirect(&rth->dst); 2487 2488 return rth; 2489 } 2490 2491 /* 2492 * Major route resolver routine. 2493 */ 2494 2495 struct rtable *ip_route_output_key_hash(struct net *net, struct flowi4 *fl4, 2496 const struct sk_buff *skb) 2497 { 2498 __u8 tos = RT_FL_TOS(fl4); 2499 struct fib_result res = { 2500 .type = RTN_UNSPEC, 2501 .fi = NULL, 2502 .table = NULL, 2503 .tclassid = 0, 2504 }; 2505 struct rtable *rth; 2506 2507 fl4->flowi4_iif = LOOPBACK_IFINDEX; 2508 fl4->flowi4_tos = tos & IPTOS_RT_MASK; 2509 fl4->flowi4_scope = ((tos & RTO_ONLINK) ? 2510 RT_SCOPE_LINK : RT_SCOPE_UNIVERSE); 2511 2512 rcu_read_lock(); 2513 rth = ip_route_output_key_hash_rcu(net, fl4, &res, skb); 2514 rcu_read_unlock(); 2515 2516 return rth; 2517 } 2518 EXPORT_SYMBOL_GPL(ip_route_output_key_hash); 2519 2520 struct rtable *ip_route_output_key_hash_rcu(struct net *net, struct flowi4 *fl4, 2521 struct fib_result *res, 2522 const struct sk_buff *skb) 2523 { 2524 struct net_device *dev_out = NULL; 2525 int orig_oif = fl4->flowi4_oif; 2526 unsigned int flags = 0; 2527 struct rtable *rth; 2528 int err; 2529 2530 if (fl4->saddr) { 2531 if (ipv4_is_multicast(fl4->saddr) || 2532 ipv4_is_lbcast(fl4->saddr) || 2533 ipv4_is_zeronet(fl4->saddr)) { 2534 rth = ERR_PTR(-EINVAL); 2535 goto out; 2536 } 2537 2538 rth = ERR_PTR(-ENETUNREACH); 2539 2540 /* I removed check for oif == dev_out->oif here. 2541 * It was wrong for two reasons: 2542 * 1. ip_dev_find(net, saddr) can return wrong iface, if saddr 2543 * is assigned to multiple interfaces. 2544 * 2. Moreover, we are allowed to send packets with saddr 2545 * of another iface. --ANK 2546 */ 2547 2548 if (fl4->flowi4_oif == 0 && 2549 (ipv4_is_multicast(fl4->daddr) || 2550 ipv4_is_lbcast(fl4->daddr))) { 2551 /* It is equivalent to inet_addr_type(saddr) == RTN_LOCAL */ 2552 dev_out = __ip_dev_find(net, fl4->saddr, false); 2553 if (!dev_out) 2554 goto out; 2555 2556 /* Special hack: user can direct multicasts 2557 * and limited broadcast via necessary interface 2558 * without fiddling with IP_MULTICAST_IF or IP_PKTINFO. 2559 * This hack is not just for fun, it allows 2560 * vic,vat and friends to work. 2561 * They bind socket to loopback, set ttl to zero 2562 * and expect that it will work. 2563 * From the viewpoint of routing cache they are broken, 2564 * because we are not allowed to build multicast path 2565 * with loopback source addr (look, routing cache 2566 * cannot know, that ttl is zero, so that packet 2567 * will not leave this host and route is valid). 2568 * Luckily, this hack is good workaround. 2569 */ 2570 2571 fl4->flowi4_oif = dev_out->ifindex; 2572 goto make_route; 2573 } 2574 2575 if (!(fl4->flowi4_flags & FLOWI_FLAG_ANYSRC)) { 2576 /* It is equivalent to inet_addr_type(saddr) == RTN_LOCAL */ 2577 if (!__ip_dev_find(net, fl4->saddr, false)) 2578 goto out; 2579 } 2580 } 2581 2582 2583 if (fl4->flowi4_oif) { 2584 dev_out = dev_get_by_index_rcu(net, fl4->flowi4_oif); 2585 rth = ERR_PTR(-ENODEV); 2586 if (!dev_out) 2587 goto out; 2588 2589 /* RACE: Check return value of inet_select_addr instead. */ 2590 if (!(dev_out->flags & IFF_UP) || !__in_dev_get_rcu(dev_out)) { 2591 rth = ERR_PTR(-ENETUNREACH); 2592 goto out; 2593 } 2594 if (ipv4_is_local_multicast(fl4->daddr) || 2595 ipv4_is_lbcast(fl4->daddr) || 2596 fl4->flowi4_proto == IPPROTO_IGMP) { 2597 if (!fl4->saddr) 2598 fl4->saddr = inet_select_addr(dev_out, 0, 2599 RT_SCOPE_LINK); 2600 goto make_route; 2601 } 2602 if (!fl4->saddr) { 2603 if (ipv4_is_multicast(fl4->daddr)) 2604 fl4->saddr = inet_select_addr(dev_out, 0, 2605 fl4->flowi4_scope); 2606 else if (!fl4->daddr) 2607 fl4->saddr = inet_select_addr(dev_out, 0, 2608 RT_SCOPE_HOST); 2609 } 2610 } 2611 2612 if (!fl4->daddr) { 2613 fl4->daddr = fl4->saddr; 2614 if (!fl4->daddr) 2615 fl4->daddr = fl4->saddr = htonl(INADDR_LOOPBACK); 2616 dev_out = net->loopback_dev; 2617 fl4->flowi4_oif = LOOPBACK_IFINDEX; 2618 res->type = RTN_LOCAL; 2619 flags |= RTCF_LOCAL; 2620 goto make_route; 2621 } 2622 2623 err = fib_lookup(net, fl4, res, 0); 2624 if (err) { 2625 res->fi = NULL; 2626 res->table = NULL; 2627 if (fl4->flowi4_oif && 2628 (ipv4_is_multicast(fl4->daddr) || 2629 !netif_index_is_l3_master(net, fl4->flowi4_oif))) { 2630 /* Apparently, routing tables are wrong. Assume, 2631 * that the destination is on link. 2632 * 2633 * WHY? DW. 2634 * Because we are allowed to send to iface 2635 * even if it has NO routes and NO assigned 2636 * addresses. When oif is specified, routing 2637 * tables are looked up with only one purpose: 2638 * to catch if destination is gatewayed, rather than 2639 * direct. Moreover, if MSG_DONTROUTE is set, 2640 * we send packet, ignoring both routing tables 2641 * and ifaddr state. --ANK 2642 * 2643 * 2644 * We could make it even if oif is unknown, 2645 * likely IPv6, but we do not. 2646 */ 2647 2648 if (fl4->saddr == 0) 2649 fl4->saddr = inet_select_addr(dev_out, 0, 2650 RT_SCOPE_LINK); 2651 res->type = RTN_UNICAST; 2652 goto make_route; 2653 } 2654 rth = ERR_PTR(err); 2655 goto out; 2656 } 2657 2658 if (res->type == RTN_LOCAL) { 2659 if (!fl4->saddr) { 2660 if (res->fi->fib_prefsrc) 2661 fl4->saddr = res->fi->fib_prefsrc; 2662 else 2663 fl4->saddr = fl4->daddr; 2664 } 2665 2666 /* L3 master device is the loopback for that domain */ 2667 dev_out = l3mdev_master_dev_rcu(FIB_RES_DEV(*res)) ? : 2668 net->loopback_dev; 2669 2670 /* make sure orig_oif points to fib result device even 2671 * though packet rx/tx happens over loopback or l3mdev 2672 */ 2673 orig_oif = FIB_RES_OIF(*res); 2674 2675 fl4->flowi4_oif = dev_out->ifindex; 2676 flags |= RTCF_LOCAL; 2677 goto make_route; 2678 } 2679 2680 fib_select_path(net, res, fl4, skb); 2681 2682 dev_out = FIB_RES_DEV(*res); 2683 2684 make_route: 2685 rth = __mkroute_output(res, fl4, orig_oif, dev_out, flags); 2686 2687 out: 2688 return rth; 2689 } 2690 2691 static struct dst_entry *ipv4_blackhole_dst_check(struct dst_entry *dst, u32 cookie) 2692 { 2693 return NULL; 2694 } 2695 2696 static unsigned int ipv4_blackhole_mtu(const struct dst_entry *dst) 2697 { 2698 unsigned int mtu = dst_metric_raw(dst, RTAX_MTU); 2699 2700 return mtu ? : dst->dev->mtu; 2701 } 2702 2703 static void ipv4_rt_blackhole_update_pmtu(struct dst_entry *dst, struct sock *sk, 2704 struct sk_buff *skb, u32 mtu, 2705 bool confirm_neigh) 2706 { 2707 } 2708 2709 static void ipv4_rt_blackhole_redirect(struct dst_entry *dst, struct sock *sk, 2710 struct sk_buff *skb) 2711 { 2712 } 2713 2714 static u32 *ipv4_rt_blackhole_cow_metrics(struct dst_entry *dst, 2715 unsigned long old) 2716 { 2717 return NULL; 2718 } 2719 2720 static struct dst_ops ipv4_dst_blackhole_ops = { 2721 .family = AF_INET, 2722 .check = ipv4_blackhole_dst_check, 2723 .mtu = ipv4_blackhole_mtu, 2724 .default_advmss = ipv4_default_advmss, 2725 .update_pmtu = ipv4_rt_blackhole_update_pmtu, 2726 .redirect = ipv4_rt_blackhole_redirect, 2727 .cow_metrics = ipv4_rt_blackhole_cow_metrics, 2728 .neigh_lookup = ipv4_neigh_lookup, 2729 }; 2730 2731 struct dst_entry *ipv4_blackhole_route(struct net *net, struct dst_entry *dst_orig) 2732 { 2733 struct rtable *ort = (struct rtable *) dst_orig; 2734 struct rtable *rt; 2735 2736 rt = dst_alloc(&ipv4_dst_blackhole_ops, NULL, 1, DST_OBSOLETE_DEAD, 0); 2737 if (rt) { 2738 struct dst_entry *new = &rt->dst; 2739 2740 new->__use = 1; 2741 new->input = dst_discard; 2742 new->output = dst_discard_out; 2743 2744 new->dev = net->loopback_dev; 2745 if (new->dev) 2746 dev_hold(new->dev); 2747 2748 rt->rt_is_input = ort->rt_is_input; 2749 rt->rt_iif = ort->rt_iif; 2750 rt->rt_pmtu = ort->rt_pmtu; 2751 rt->rt_mtu_locked = ort->rt_mtu_locked; 2752 2753 rt->rt_genid = rt_genid_ipv4(net); 2754 rt->rt_flags = ort->rt_flags; 2755 rt->rt_type = ort->rt_type; 2756 rt->rt_uses_gateway = ort->rt_uses_gateway; 2757 rt->rt_gw_family = ort->rt_gw_family; 2758 if (rt->rt_gw_family == AF_INET) 2759 rt->rt_gw4 = ort->rt_gw4; 2760 else if (rt->rt_gw_family == AF_INET6) 2761 rt->rt_gw6 = ort->rt_gw6; 2762 2763 INIT_LIST_HEAD(&rt->rt_uncached); 2764 } 2765 2766 dst_release(dst_orig); 2767 2768 return rt ? &rt->dst : ERR_PTR(-ENOMEM); 2769 } 2770 2771 struct rtable *ip_route_output_flow(struct net *net, struct flowi4 *flp4, 2772 const struct sock *sk) 2773 { 2774 struct rtable *rt = __ip_route_output_key(net, flp4); 2775 2776 if (IS_ERR(rt)) 2777 return rt; 2778 2779 if (flp4->flowi4_proto) { 2780 flp4->flowi4_oif = rt->dst.dev->ifindex; 2781 rt = (struct rtable *)xfrm_lookup_route(net, &rt->dst, 2782 flowi4_to_flowi(flp4), 2783 sk, 0); 2784 } 2785 2786 return rt; 2787 } 2788 EXPORT_SYMBOL_GPL(ip_route_output_flow); 2789 2790 struct rtable *ip_route_output_tunnel(struct sk_buff *skb, 2791 struct net_device *dev, 2792 struct net *net, __be32 *saddr, 2793 const struct ip_tunnel_info *info, 2794 u8 protocol, bool use_cache) 2795 { 2796 #ifdef CONFIG_DST_CACHE 2797 struct dst_cache *dst_cache; 2798 #endif 2799 struct rtable *rt = NULL; 2800 struct flowi4 fl4; 2801 __u8 tos; 2802 2803 #ifdef CONFIG_DST_CACHE 2804 dst_cache = (struct dst_cache *)&info->dst_cache; 2805 if (use_cache) { 2806 rt = dst_cache_get_ip4(dst_cache, saddr); 2807 if (rt) 2808 return rt; 2809 } 2810 #endif 2811 memset(&fl4, 0, sizeof(fl4)); 2812 fl4.flowi4_mark = skb->mark; 2813 fl4.flowi4_proto = protocol; 2814 fl4.daddr = info->key.u.ipv4.dst; 2815 fl4.saddr = info->key.u.ipv4.src; 2816 tos = info->key.tos; 2817 fl4.flowi4_tos = RT_TOS(tos); 2818 2819 rt = ip_route_output_key(net, &fl4); 2820 if (IS_ERR(rt)) { 2821 netdev_dbg(dev, "no route to %pI4\n", &fl4.daddr); 2822 return ERR_PTR(-ENETUNREACH); 2823 } 2824 if (rt->dst.dev == dev) { /* is this necessary? */ 2825 netdev_dbg(dev, "circular route to %pI4\n", &fl4.daddr); 2826 ip_rt_put(rt); 2827 return ERR_PTR(-ELOOP); 2828 } 2829 #ifdef CONFIG_DST_CACHE 2830 if (use_cache) 2831 dst_cache_set_ip4(dst_cache, &rt->dst, fl4.saddr); 2832 #endif 2833 *saddr = fl4.saddr; 2834 return rt; 2835 } 2836 EXPORT_SYMBOL_GPL(ip_route_output_tunnel); 2837 2838 /* called with rcu_read_lock held */ 2839 static int rt_fill_info(struct net *net, __be32 dst, __be32 src, 2840 struct rtable *rt, u32 table_id, struct flowi4 *fl4, 2841 struct sk_buff *skb, u32 portid, u32 seq, 2842 unsigned int flags) 2843 { 2844 struct rtmsg *r; 2845 struct nlmsghdr *nlh; 2846 unsigned long expires = 0; 2847 u32 error; 2848 u32 metrics[RTAX_MAX]; 2849 2850 nlh = nlmsg_put(skb, portid, seq, RTM_NEWROUTE, sizeof(*r), flags); 2851 if (!nlh) 2852 return -EMSGSIZE; 2853 2854 r = nlmsg_data(nlh); 2855 r->rtm_family = AF_INET; 2856 r->rtm_dst_len = 32; 2857 r->rtm_src_len = 0; 2858 r->rtm_tos = fl4 ? fl4->flowi4_tos : 0; 2859 r->rtm_table = table_id < 256 ? table_id : RT_TABLE_COMPAT; 2860 if (nla_put_u32(skb, RTA_TABLE, table_id)) 2861 goto nla_put_failure; 2862 r->rtm_type = rt->rt_type; 2863 r->rtm_scope = RT_SCOPE_UNIVERSE; 2864 r->rtm_protocol = RTPROT_UNSPEC; 2865 r->rtm_flags = (rt->rt_flags & ~0xFFFF) | RTM_F_CLONED; 2866 if (rt->rt_flags & RTCF_NOTIFY) 2867 r->rtm_flags |= RTM_F_NOTIFY; 2868 if (IPCB(skb)->flags & IPSKB_DOREDIRECT) 2869 r->rtm_flags |= RTCF_DOREDIRECT; 2870 2871 if (nla_put_in_addr(skb, RTA_DST, dst)) 2872 goto nla_put_failure; 2873 if (src) { 2874 r->rtm_src_len = 32; 2875 if (nla_put_in_addr(skb, RTA_SRC, src)) 2876 goto nla_put_failure; 2877 } 2878 if (rt->dst.dev && 2879 nla_put_u32(skb, RTA_OIF, rt->dst.dev->ifindex)) 2880 goto nla_put_failure; 2881 if (rt->dst.lwtstate && 2882 lwtunnel_fill_encap(skb, rt->dst.lwtstate, RTA_ENCAP, RTA_ENCAP_TYPE) < 0) 2883 goto nla_put_failure; 2884 #ifdef CONFIG_IP_ROUTE_CLASSID 2885 if (rt->dst.tclassid && 2886 nla_put_u32(skb, RTA_FLOW, rt->dst.tclassid)) 2887 goto nla_put_failure; 2888 #endif 2889 if (fl4 && !rt_is_input_route(rt) && 2890 fl4->saddr != src) { 2891 if (nla_put_in_addr(skb, RTA_PREFSRC, fl4->saddr)) 2892 goto nla_put_failure; 2893 } 2894 if (rt->rt_uses_gateway) { 2895 if (rt->rt_gw_family == AF_INET && 2896 nla_put_in_addr(skb, RTA_GATEWAY, rt->rt_gw4)) { 2897 goto nla_put_failure; 2898 } else if (rt->rt_gw_family == AF_INET6) { 2899 int alen = sizeof(struct in6_addr); 2900 struct nlattr *nla; 2901 struct rtvia *via; 2902 2903 nla = nla_reserve(skb, RTA_VIA, alen + 2); 2904 if (!nla) 2905 goto nla_put_failure; 2906 2907 via = nla_data(nla); 2908 via->rtvia_family = AF_INET6; 2909 memcpy(via->rtvia_addr, &rt->rt_gw6, alen); 2910 } 2911 } 2912 2913 expires = rt->dst.expires; 2914 if (expires) { 2915 unsigned long now = jiffies; 2916 2917 if (time_before(now, expires)) 2918 expires -= now; 2919 else 2920 expires = 0; 2921 } 2922 2923 memcpy(metrics, dst_metrics_ptr(&rt->dst), sizeof(metrics)); 2924 if (rt->rt_pmtu && expires) 2925 metrics[RTAX_MTU - 1] = rt->rt_pmtu; 2926 if (rt->rt_mtu_locked && expires) 2927 metrics[RTAX_LOCK - 1] |= BIT(RTAX_MTU); 2928 if (rtnetlink_put_metrics(skb, metrics) < 0) 2929 goto nla_put_failure; 2930 2931 if (fl4) { 2932 if (fl4->flowi4_mark && 2933 nla_put_u32(skb, RTA_MARK, fl4->flowi4_mark)) 2934 goto nla_put_failure; 2935 2936 if (!uid_eq(fl4->flowi4_uid, INVALID_UID) && 2937 nla_put_u32(skb, RTA_UID, 2938 from_kuid_munged(current_user_ns(), 2939 fl4->flowi4_uid))) 2940 goto nla_put_failure; 2941 2942 if (rt_is_input_route(rt)) { 2943 #ifdef CONFIG_IP_MROUTE 2944 if (ipv4_is_multicast(dst) && 2945 !ipv4_is_local_multicast(dst) && 2946 IPV4_DEVCONF_ALL(net, MC_FORWARDING)) { 2947 int err = ipmr_get_route(net, skb, 2948 fl4->saddr, fl4->daddr, 2949 r, portid); 2950 2951 if (err <= 0) { 2952 if (err == 0) 2953 return 0; 2954 goto nla_put_failure; 2955 } 2956 } else 2957 #endif 2958 if (nla_put_u32(skb, RTA_IIF, fl4->flowi4_iif)) 2959 goto nla_put_failure; 2960 } 2961 } 2962 2963 error = rt->dst.error; 2964 2965 if (rtnl_put_cacheinfo(skb, &rt->dst, 0, expires, error) < 0) 2966 goto nla_put_failure; 2967 2968 nlmsg_end(skb, nlh); 2969 return 0; 2970 2971 nla_put_failure: 2972 nlmsg_cancel(skb, nlh); 2973 return -EMSGSIZE; 2974 } 2975 2976 static int fnhe_dump_bucket(struct net *net, struct sk_buff *skb, 2977 struct netlink_callback *cb, u32 table_id, 2978 struct fnhe_hash_bucket *bucket, int genid, 2979 int *fa_index, int fa_start, unsigned int flags) 2980 { 2981 int i; 2982 2983 for (i = 0; i < FNHE_HASH_SIZE; i++) { 2984 struct fib_nh_exception *fnhe; 2985 2986 for (fnhe = rcu_dereference(bucket[i].chain); fnhe; 2987 fnhe = rcu_dereference(fnhe->fnhe_next)) { 2988 struct rtable *rt; 2989 int err; 2990 2991 if (*fa_index < fa_start) 2992 goto next; 2993 2994 if (fnhe->fnhe_genid != genid) 2995 goto next; 2996 2997 if (fnhe->fnhe_expires && 2998 time_after(jiffies, fnhe->fnhe_expires)) 2999 goto next; 3000 3001 rt = rcu_dereference(fnhe->fnhe_rth_input); 3002 if (!rt) 3003 rt = rcu_dereference(fnhe->fnhe_rth_output); 3004 if (!rt) 3005 goto next; 3006 3007 err = rt_fill_info(net, fnhe->fnhe_daddr, 0, rt, 3008 table_id, NULL, skb, 3009 NETLINK_CB(cb->skb).portid, 3010 cb->nlh->nlmsg_seq, flags); 3011 if (err) 3012 return err; 3013 next: 3014 (*fa_index)++; 3015 } 3016 } 3017 3018 return 0; 3019 } 3020 3021 int fib_dump_info_fnhe(struct sk_buff *skb, struct netlink_callback *cb, 3022 u32 table_id, struct fib_info *fi, 3023 int *fa_index, int fa_start, unsigned int flags) 3024 { 3025 struct net *net = sock_net(cb->skb->sk); 3026 int nhsel, genid = fnhe_genid(net); 3027 3028 for (nhsel = 0; nhsel < fib_info_num_path(fi); nhsel++) { 3029 struct fib_nh_common *nhc = fib_info_nhc(fi, nhsel); 3030 struct fnhe_hash_bucket *bucket; 3031 int err; 3032 3033 if (nhc->nhc_flags & RTNH_F_DEAD) 3034 continue; 3035 3036 rcu_read_lock(); 3037 bucket = rcu_dereference(nhc->nhc_exceptions); 3038 err = 0; 3039 if (bucket) 3040 err = fnhe_dump_bucket(net, skb, cb, table_id, bucket, 3041 genid, fa_index, fa_start, 3042 flags); 3043 rcu_read_unlock(); 3044 if (err) 3045 return err; 3046 } 3047 3048 return 0; 3049 } 3050 3051 static struct sk_buff *inet_rtm_getroute_build_skb(__be32 src, __be32 dst, 3052 u8 ip_proto, __be16 sport, 3053 __be16 dport) 3054 { 3055 struct sk_buff *skb; 3056 struct iphdr *iph; 3057 3058 skb = alloc_skb(NLMSG_GOODSIZE, GFP_KERNEL); 3059 if (!skb) 3060 return NULL; 3061 3062 /* Reserve room for dummy headers, this skb can pass 3063 * through good chunk of routing engine. 3064 */ 3065 skb_reset_mac_header(skb); 3066 skb_reset_network_header(skb); 3067 skb->protocol = htons(ETH_P_IP); 3068 iph = skb_put(skb, sizeof(struct iphdr)); 3069 iph->protocol = ip_proto; 3070 iph->saddr = src; 3071 iph->daddr = dst; 3072 iph->version = 0x4; 3073 iph->frag_off = 0; 3074 iph->ihl = 0x5; 3075 skb_set_transport_header(skb, skb->len); 3076 3077 switch (iph->protocol) { 3078 case IPPROTO_UDP: { 3079 struct udphdr *udph; 3080 3081 udph = skb_put_zero(skb, sizeof(struct udphdr)); 3082 udph->source = sport; 3083 udph->dest = dport; 3084 udph->len = sizeof(struct udphdr); 3085 udph->check = 0; 3086 break; 3087 } 3088 case IPPROTO_TCP: { 3089 struct tcphdr *tcph; 3090 3091 tcph = skb_put_zero(skb, sizeof(struct tcphdr)); 3092 tcph->source = sport; 3093 tcph->dest = dport; 3094 tcph->doff = sizeof(struct tcphdr) / 4; 3095 tcph->rst = 1; 3096 tcph->check = ~tcp_v4_check(sizeof(struct tcphdr), 3097 src, dst, 0); 3098 break; 3099 } 3100 case IPPROTO_ICMP: { 3101 struct icmphdr *icmph; 3102 3103 icmph = skb_put_zero(skb, sizeof(struct icmphdr)); 3104 icmph->type = ICMP_ECHO; 3105 icmph->code = 0; 3106 } 3107 } 3108 3109 return skb; 3110 } 3111 3112 static int inet_rtm_valid_getroute_req(struct sk_buff *skb, 3113 const struct nlmsghdr *nlh, 3114 struct nlattr **tb, 3115 struct netlink_ext_ack *extack) 3116 { 3117 struct rtmsg *rtm; 3118 int i, err; 3119 3120 if (nlh->nlmsg_len < nlmsg_msg_size(sizeof(*rtm))) { 3121 NL_SET_ERR_MSG(extack, 3122 "ipv4: Invalid header for route get request"); 3123 return -EINVAL; 3124 } 3125 3126 if (!netlink_strict_get_check(skb)) 3127 return nlmsg_parse_deprecated(nlh, sizeof(*rtm), tb, RTA_MAX, 3128 rtm_ipv4_policy, extack); 3129 3130 rtm = nlmsg_data(nlh); 3131 if ((rtm->rtm_src_len && rtm->rtm_src_len != 32) || 3132 (rtm->rtm_dst_len && rtm->rtm_dst_len != 32) || 3133 rtm->rtm_table || rtm->rtm_protocol || 3134 rtm->rtm_scope || rtm->rtm_type) { 3135 NL_SET_ERR_MSG(extack, "ipv4: Invalid values in header for route get request"); 3136 return -EINVAL; 3137 } 3138 3139 if (rtm->rtm_flags & ~(RTM_F_NOTIFY | 3140 RTM_F_LOOKUP_TABLE | 3141 RTM_F_FIB_MATCH)) { 3142 NL_SET_ERR_MSG(extack, "ipv4: Unsupported rtm_flags for route get request"); 3143 return -EINVAL; 3144 } 3145 3146 err = nlmsg_parse_deprecated_strict(nlh, sizeof(*rtm), tb, RTA_MAX, 3147 rtm_ipv4_policy, extack); 3148 if (err) 3149 return err; 3150 3151 if ((tb[RTA_SRC] && !rtm->rtm_src_len) || 3152 (tb[RTA_DST] && !rtm->rtm_dst_len)) { 3153 NL_SET_ERR_MSG(extack, "ipv4: rtm_src_len and rtm_dst_len must be 32 for IPv4"); 3154 return -EINVAL; 3155 } 3156 3157 for (i = 0; i <= RTA_MAX; i++) { 3158 if (!tb[i]) 3159 continue; 3160 3161 switch (i) { 3162 case RTA_IIF: 3163 case RTA_OIF: 3164 case RTA_SRC: 3165 case RTA_DST: 3166 case RTA_IP_PROTO: 3167 case RTA_SPORT: 3168 case RTA_DPORT: 3169 case RTA_MARK: 3170 case RTA_UID: 3171 break; 3172 default: 3173 NL_SET_ERR_MSG(extack, "ipv4: Unsupported attribute in route get request"); 3174 return -EINVAL; 3175 } 3176 } 3177 3178 return 0; 3179 } 3180 3181 static int inet_rtm_getroute(struct sk_buff *in_skb, struct nlmsghdr *nlh, 3182 struct netlink_ext_ack *extack) 3183 { 3184 struct net *net = sock_net(in_skb->sk); 3185 struct nlattr *tb[RTA_MAX+1]; 3186 u32 table_id = RT_TABLE_MAIN; 3187 __be16 sport = 0, dport = 0; 3188 struct fib_result res = {}; 3189 u8 ip_proto = IPPROTO_UDP; 3190 struct rtable *rt = NULL; 3191 struct sk_buff *skb; 3192 struct rtmsg *rtm; 3193 struct flowi4 fl4 = {}; 3194 __be32 dst = 0; 3195 __be32 src = 0; 3196 kuid_t uid; 3197 u32 iif; 3198 int err; 3199 int mark; 3200 3201 err = inet_rtm_valid_getroute_req(in_skb, nlh, tb, extack); 3202 if (err < 0) 3203 return err; 3204 3205 rtm = nlmsg_data(nlh); 3206 src = tb[RTA_SRC] ? nla_get_in_addr(tb[RTA_SRC]) : 0; 3207 dst = tb[RTA_DST] ? nla_get_in_addr(tb[RTA_DST]) : 0; 3208 iif = tb[RTA_IIF] ? nla_get_u32(tb[RTA_IIF]) : 0; 3209 mark = tb[RTA_MARK] ? nla_get_u32(tb[RTA_MARK]) : 0; 3210 if (tb[RTA_UID]) 3211 uid = make_kuid(current_user_ns(), nla_get_u32(tb[RTA_UID])); 3212 else 3213 uid = (iif ? INVALID_UID : current_uid()); 3214 3215 if (tb[RTA_IP_PROTO]) { 3216 err = rtm_getroute_parse_ip_proto(tb[RTA_IP_PROTO], 3217 &ip_proto, AF_INET, extack); 3218 if (err) 3219 return err; 3220 } 3221 3222 if (tb[RTA_SPORT]) 3223 sport = nla_get_be16(tb[RTA_SPORT]); 3224 3225 if (tb[RTA_DPORT]) 3226 dport = nla_get_be16(tb[RTA_DPORT]); 3227 3228 skb = inet_rtm_getroute_build_skb(src, dst, ip_proto, sport, dport); 3229 if (!skb) 3230 return -ENOBUFS; 3231 3232 fl4.daddr = dst; 3233 fl4.saddr = src; 3234 fl4.flowi4_tos = rtm->rtm_tos & IPTOS_RT_MASK; 3235 fl4.flowi4_oif = tb[RTA_OIF] ? nla_get_u32(tb[RTA_OIF]) : 0; 3236 fl4.flowi4_mark = mark; 3237 fl4.flowi4_uid = uid; 3238 if (sport) 3239 fl4.fl4_sport = sport; 3240 if (dport) 3241 fl4.fl4_dport = dport; 3242 fl4.flowi4_proto = ip_proto; 3243 3244 rcu_read_lock(); 3245 3246 if (iif) { 3247 struct net_device *dev; 3248 3249 dev = dev_get_by_index_rcu(net, iif); 3250 if (!dev) { 3251 err = -ENODEV; 3252 goto errout_rcu; 3253 } 3254 3255 fl4.flowi4_iif = iif; /* for rt_fill_info */ 3256 skb->dev = dev; 3257 skb->mark = mark; 3258 err = ip_route_input_rcu(skb, dst, src, 3259 rtm->rtm_tos & IPTOS_RT_MASK, dev, 3260 &res); 3261 3262 rt = skb_rtable(skb); 3263 if (err == 0 && rt->dst.error) 3264 err = -rt->dst.error; 3265 } else { 3266 fl4.flowi4_iif = LOOPBACK_IFINDEX; 3267 skb->dev = net->loopback_dev; 3268 rt = ip_route_output_key_hash_rcu(net, &fl4, &res, skb); 3269 err = 0; 3270 if (IS_ERR(rt)) 3271 err = PTR_ERR(rt); 3272 else 3273 skb_dst_set(skb, &rt->dst); 3274 } 3275 3276 if (err) 3277 goto errout_rcu; 3278 3279 if (rtm->rtm_flags & RTM_F_NOTIFY) 3280 rt->rt_flags |= RTCF_NOTIFY; 3281 3282 if (rtm->rtm_flags & RTM_F_LOOKUP_TABLE) 3283 table_id = res.table ? res.table->tb_id : 0; 3284 3285 /* reset skb for netlink reply msg */ 3286 skb_trim(skb, 0); 3287 skb_reset_network_header(skb); 3288 skb_reset_transport_header(skb); 3289 skb_reset_mac_header(skb); 3290 3291 if (rtm->rtm_flags & RTM_F_FIB_MATCH) { 3292 struct fib_rt_info fri; 3293 3294 if (!res.fi) { 3295 err = fib_props[res.type].error; 3296 if (!err) 3297 err = -EHOSTUNREACH; 3298 goto errout_rcu; 3299 } 3300 fri.fi = res.fi; 3301 fri.tb_id = table_id; 3302 fri.dst = res.prefix; 3303 fri.dst_len = res.prefixlen; 3304 fri.tos = fl4.flowi4_tos; 3305 fri.type = rt->rt_type; 3306 fri.offload = 0; 3307 fri.trap = 0; 3308 fri.offload_failed = 0; 3309 if (res.fa_head) { 3310 struct fib_alias *fa; 3311 3312 hlist_for_each_entry_rcu(fa, res.fa_head, fa_list) { 3313 u8 slen = 32 - fri.dst_len; 3314 3315 if (fa->fa_slen == slen && 3316 fa->tb_id == fri.tb_id && 3317 fa->fa_tos == fri.tos && 3318 fa->fa_info == res.fi && 3319 fa->fa_type == fri.type) { 3320 fri.offload = fa->offload; 3321 fri.trap = fa->trap; 3322 break; 3323 } 3324 } 3325 } 3326 err = fib_dump_info(skb, NETLINK_CB(in_skb).portid, 3327 nlh->nlmsg_seq, RTM_NEWROUTE, &fri, 0); 3328 } else { 3329 err = rt_fill_info(net, dst, src, rt, table_id, &fl4, skb, 3330 NETLINK_CB(in_skb).portid, 3331 nlh->nlmsg_seq, 0); 3332 } 3333 if (err < 0) 3334 goto errout_rcu; 3335 3336 rcu_read_unlock(); 3337 3338 err = rtnl_unicast(skb, net, NETLINK_CB(in_skb).portid); 3339 3340 errout_free: 3341 return err; 3342 errout_rcu: 3343 rcu_read_unlock(); 3344 kfree_skb(skb); 3345 goto errout_free; 3346 } 3347 3348 void ip_rt_multicast_event(struct in_device *in_dev) 3349 { 3350 rt_cache_flush(dev_net(in_dev->dev)); 3351 } 3352 3353 #ifdef CONFIG_SYSCTL 3354 static int ip_rt_gc_interval __read_mostly = 60 * HZ; 3355 static int ip_rt_gc_min_interval __read_mostly = HZ / 2; 3356 static int ip_rt_gc_elasticity __read_mostly = 8; 3357 static int ip_min_valid_pmtu __read_mostly = IPV4_MIN_MTU; 3358 3359 static int ipv4_sysctl_rtcache_flush(struct ctl_table *__ctl, int write, 3360 void *buffer, size_t *lenp, loff_t *ppos) 3361 { 3362 struct net *net = (struct net *)__ctl->extra1; 3363 3364 if (write) { 3365 rt_cache_flush(net); 3366 fnhe_genid_bump(net); 3367 return 0; 3368 } 3369 3370 return -EINVAL; 3371 } 3372 3373 static struct ctl_table ipv4_route_table[] = { 3374 { 3375 .procname = "gc_thresh", 3376 .data = &ipv4_dst_ops.gc_thresh, 3377 .maxlen = sizeof(int), 3378 .mode = 0644, 3379 .proc_handler = proc_dointvec, 3380 }, 3381 { 3382 .procname = "max_size", 3383 .data = &ip_rt_max_size, 3384 .maxlen = sizeof(int), 3385 .mode = 0644, 3386 .proc_handler = proc_dointvec, 3387 }, 3388 { 3389 /* Deprecated. Use gc_min_interval_ms */ 3390 3391 .procname = "gc_min_interval", 3392 .data = &ip_rt_gc_min_interval, 3393 .maxlen = sizeof(int), 3394 .mode = 0644, 3395 .proc_handler = proc_dointvec_jiffies, 3396 }, 3397 { 3398 .procname = "gc_min_interval_ms", 3399 .data = &ip_rt_gc_min_interval, 3400 .maxlen = sizeof(int), 3401 .mode = 0644, 3402 .proc_handler = proc_dointvec_ms_jiffies, 3403 }, 3404 { 3405 .procname = "gc_timeout", 3406 .data = &ip_rt_gc_timeout, 3407 .maxlen = sizeof(int), 3408 .mode = 0644, 3409 .proc_handler = proc_dointvec_jiffies, 3410 }, 3411 { 3412 .procname = "gc_interval", 3413 .data = &ip_rt_gc_interval, 3414 .maxlen = sizeof(int), 3415 .mode = 0644, 3416 .proc_handler = proc_dointvec_jiffies, 3417 }, 3418 { 3419 .procname = "redirect_load", 3420 .data = &ip_rt_redirect_load, 3421 .maxlen = sizeof(int), 3422 .mode = 0644, 3423 .proc_handler = proc_dointvec, 3424 }, 3425 { 3426 .procname = "redirect_number", 3427 .data = &ip_rt_redirect_number, 3428 .maxlen = sizeof(int), 3429 .mode = 0644, 3430 .proc_handler = proc_dointvec, 3431 }, 3432 { 3433 .procname = "redirect_silence", 3434 .data = &ip_rt_redirect_silence, 3435 .maxlen = sizeof(int), 3436 .mode = 0644, 3437 .proc_handler = proc_dointvec, 3438 }, 3439 { 3440 .procname = "error_cost", 3441 .data = &ip_rt_error_cost, 3442 .maxlen = sizeof(int), 3443 .mode = 0644, 3444 .proc_handler = proc_dointvec, 3445 }, 3446 { 3447 .procname = "error_burst", 3448 .data = &ip_rt_error_burst, 3449 .maxlen = sizeof(int), 3450 .mode = 0644, 3451 .proc_handler = proc_dointvec, 3452 }, 3453 { 3454 .procname = "gc_elasticity", 3455 .data = &ip_rt_gc_elasticity, 3456 .maxlen = sizeof(int), 3457 .mode = 0644, 3458 .proc_handler = proc_dointvec, 3459 }, 3460 { 3461 .procname = "mtu_expires", 3462 .data = &ip_rt_mtu_expires, 3463 .maxlen = sizeof(int), 3464 .mode = 0644, 3465 .proc_handler = proc_dointvec_jiffies, 3466 }, 3467 { 3468 .procname = "min_pmtu", 3469 .data = &ip_rt_min_pmtu, 3470 .maxlen = sizeof(int), 3471 .mode = 0644, 3472 .proc_handler = proc_dointvec_minmax, 3473 .extra1 = &ip_min_valid_pmtu, 3474 }, 3475 { 3476 .procname = "min_adv_mss", 3477 .data = &ip_rt_min_advmss, 3478 .maxlen = sizeof(int), 3479 .mode = 0644, 3480 .proc_handler = proc_dointvec, 3481 }, 3482 { } 3483 }; 3484 3485 static const char ipv4_route_flush_procname[] = "flush"; 3486 3487 static struct ctl_table ipv4_route_flush_table[] = { 3488 { 3489 .procname = ipv4_route_flush_procname, 3490 .maxlen = sizeof(int), 3491 .mode = 0200, 3492 .proc_handler = ipv4_sysctl_rtcache_flush, 3493 }, 3494 { }, 3495 }; 3496 3497 static __net_init int sysctl_route_net_init(struct net *net) 3498 { 3499 struct ctl_table *tbl; 3500 3501 tbl = ipv4_route_flush_table; 3502 if (!net_eq(net, &init_net)) { 3503 tbl = kmemdup(tbl, sizeof(ipv4_route_flush_table), GFP_KERNEL); 3504 if (!tbl) 3505 goto err_dup; 3506 3507 /* Don't export non-whitelisted sysctls to unprivileged users */ 3508 if (net->user_ns != &init_user_ns) { 3509 if (tbl[0].procname != ipv4_route_flush_procname) 3510 tbl[0].procname = NULL; 3511 } 3512 } 3513 tbl[0].extra1 = net; 3514 3515 net->ipv4.route_hdr = register_net_sysctl(net, "net/ipv4/route", tbl); 3516 if (!net->ipv4.route_hdr) 3517 goto err_reg; 3518 return 0; 3519 3520 err_reg: 3521 if (tbl != ipv4_route_flush_table) 3522 kfree(tbl); 3523 err_dup: 3524 return -ENOMEM; 3525 } 3526 3527 static __net_exit void sysctl_route_net_exit(struct net *net) 3528 { 3529 struct ctl_table *tbl; 3530 3531 tbl = net->ipv4.route_hdr->ctl_table_arg; 3532 unregister_net_sysctl_table(net->ipv4.route_hdr); 3533 BUG_ON(tbl == ipv4_route_flush_table); 3534 kfree(tbl); 3535 } 3536 3537 static __net_initdata struct pernet_operations sysctl_route_ops = { 3538 .init = sysctl_route_net_init, 3539 .exit = sysctl_route_net_exit, 3540 }; 3541 #endif 3542 3543 static __net_init int rt_genid_init(struct net *net) 3544 { 3545 atomic_set(&net->ipv4.rt_genid, 0); 3546 atomic_set(&net->fnhe_genid, 0); 3547 atomic_set(&net->ipv4.dev_addr_genid, get_random_int()); 3548 return 0; 3549 } 3550 3551 static __net_initdata struct pernet_operations rt_genid_ops = { 3552 .init = rt_genid_init, 3553 }; 3554 3555 static int __net_init ipv4_inetpeer_init(struct net *net) 3556 { 3557 struct inet_peer_base *bp = kmalloc(sizeof(*bp), GFP_KERNEL); 3558 3559 if (!bp) 3560 return -ENOMEM; 3561 inet_peer_base_init(bp); 3562 net->ipv4.peers = bp; 3563 return 0; 3564 } 3565 3566 static void __net_exit ipv4_inetpeer_exit(struct net *net) 3567 { 3568 struct inet_peer_base *bp = net->ipv4.peers; 3569 3570 net->ipv4.peers = NULL; 3571 inetpeer_invalidate_tree(bp); 3572 kfree(bp); 3573 } 3574 3575 static __net_initdata struct pernet_operations ipv4_inetpeer_ops = { 3576 .init = ipv4_inetpeer_init, 3577 .exit = ipv4_inetpeer_exit, 3578 }; 3579 3580 #ifdef CONFIG_IP_ROUTE_CLASSID 3581 struct ip_rt_acct __percpu *ip_rt_acct __read_mostly; 3582 #endif /* CONFIG_IP_ROUTE_CLASSID */ 3583 3584 int __init ip_rt_init(void) 3585 { 3586 int cpu; 3587 3588 ip_idents = kmalloc_array(IP_IDENTS_SZ, sizeof(*ip_idents), 3589 GFP_KERNEL); 3590 if (!ip_idents) 3591 panic("IP: failed to allocate ip_idents\n"); 3592 3593 prandom_bytes(ip_idents, IP_IDENTS_SZ * sizeof(*ip_idents)); 3594 3595 ip_tstamps = kcalloc(IP_IDENTS_SZ, sizeof(*ip_tstamps), GFP_KERNEL); 3596 if (!ip_tstamps) 3597 panic("IP: failed to allocate ip_tstamps\n"); 3598 3599 for_each_possible_cpu(cpu) { 3600 struct uncached_list *ul = &per_cpu(rt_uncached_list, cpu); 3601 3602 INIT_LIST_HEAD(&ul->head); 3603 spin_lock_init(&ul->lock); 3604 } 3605 #ifdef CONFIG_IP_ROUTE_CLASSID 3606 ip_rt_acct = __alloc_percpu(256 * sizeof(struct ip_rt_acct), __alignof__(struct ip_rt_acct)); 3607 if (!ip_rt_acct) 3608 panic("IP: failed to allocate ip_rt_acct\n"); 3609 #endif 3610 3611 ipv4_dst_ops.kmem_cachep = 3612 kmem_cache_create("ip_dst_cache", sizeof(struct rtable), 0, 3613 SLAB_HWCACHE_ALIGN|SLAB_PANIC, NULL); 3614 3615 ipv4_dst_blackhole_ops.kmem_cachep = ipv4_dst_ops.kmem_cachep; 3616 3617 if (dst_entries_init(&ipv4_dst_ops) < 0) 3618 panic("IP: failed to allocate ipv4_dst_ops counter\n"); 3619 3620 if (dst_entries_init(&ipv4_dst_blackhole_ops) < 0) 3621 panic("IP: failed to allocate ipv4_dst_blackhole_ops counter\n"); 3622 3623 ipv4_dst_ops.gc_thresh = ~0; 3624 ip_rt_max_size = INT_MAX; 3625 3626 devinet_init(); 3627 ip_fib_init(); 3628 3629 if (ip_rt_proc_init()) 3630 pr_err("Unable to create route proc files\n"); 3631 #ifdef CONFIG_XFRM 3632 xfrm_init(); 3633 xfrm4_init(); 3634 #endif 3635 rtnl_register(PF_INET, RTM_GETROUTE, inet_rtm_getroute, NULL, 3636 RTNL_FLAG_DOIT_UNLOCKED); 3637 3638 #ifdef CONFIG_SYSCTL 3639 register_pernet_subsys(&sysctl_route_ops); 3640 #endif 3641 register_pernet_subsys(&rt_genid_ops); 3642 register_pernet_subsys(&ipv4_inetpeer_ops); 3643 return 0; 3644 } 3645 3646 #ifdef CONFIG_SYSCTL 3647 /* 3648 * We really need to sanitize the damn ipv4 init order, then all 3649 * this nonsense will go away. 3650 */ 3651 void __init ip_static_sysctl_init(void) 3652 { 3653 register_net_sysctl(&init_net, "net/ipv4/route", ipv4_route_table); 3654 } 3655 #endif 3656