1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * INET An implementation of the TCP/IP protocol suite for the LINUX 4 * operating system. INET is implemented using the BSD Socket 5 * interface as the means of communication with the user level. 6 * 7 * ROUTE - implementation of the IP router. 8 * 9 * Authors: Ross Biro 10 * Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG> 11 * Alan Cox, <gw4pts@gw4pts.ampr.org> 12 * Linus Torvalds, <Linus.Torvalds@helsinki.fi> 13 * Alexey Kuznetsov, <kuznet@ms2.inr.ac.ru> 14 * 15 * Fixes: 16 * Alan Cox : Verify area fixes. 17 * Alan Cox : cli() protects routing changes 18 * Rui Oliveira : ICMP routing table updates 19 * (rco@di.uminho.pt) Routing table insertion and update 20 * Linus Torvalds : Rewrote bits to be sensible 21 * Alan Cox : Added BSD route gw semantics 22 * Alan Cox : Super /proc >4K 23 * Alan Cox : MTU in route table 24 * Alan Cox : MSS actually. Also added the window 25 * clamper. 26 * Sam Lantinga : Fixed route matching in rt_del() 27 * Alan Cox : Routing cache support. 28 * Alan Cox : Removed compatibility cruft. 29 * Alan Cox : RTF_REJECT support. 30 * Alan Cox : TCP irtt support. 31 * Jonathan Naylor : Added Metric support. 32 * Miquel van Smoorenburg : BSD API fixes. 33 * Miquel van Smoorenburg : Metrics. 34 * Alan Cox : Use __u32 properly 35 * Alan Cox : Aligned routing errors more closely with BSD 36 * our system is still very different. 37 * Alan Cox : Faster /proc handling 38 * Alexey Kuznetsov : Massive rework to support tree based routing, 39 * routing caches and better behaviour. 40 * 41 * Olaf Erb : irtt wasn't being copied right. 42 * Bjorn Ekwall : Kerneld route support. 43 * Alan Cox : Multicast fixed (I hope) 44 * Pavel Krauz : Limited broadcast fixed 45 * Mike McLagan : Routing by source 46 * Alexey Kuznetsov : End of old history. Split to fib.c and 47 * route.c and rewritten from scratch. 48 * Andi Kleen : Load-limit warning messages. 49 * Vitaly E. Lavrov : Transparent proxy revived after year coma. 50 * Vitaly E. Lavrov : Race condition in ip_route_input_slow. 51 * Tobias Ringstrom : Uninitialized res.type in ip_route_output_slow. 52 * Vladimir V. Ivanov : IP rule info (flowid) is really useful. 53 * Marc Boucher : routing by fwmark 54 * Robert Olsson : Added rt_cache statistics 55 * Arnaldo C. Melo : Convert proc stuff to seq_file 56 * Eric Dumazet : hashed spinlocks and rt_check_expire() fixes. 57 * Ilia Sotnikov : Ignore TOS on PMTUD and Redirect 58 * Ilia Sotnikov : Removed TOS from hash calculations 59 */ 60 61 #define pr_fmt(fmt) "IPv4: " fmt 62 63 #include <linux/module.h> 64 #include <linux/uaccess.h> 65 #include <linux/bitops.h> 66 #include <linux/types.h> 67 #include <linux/kernel.h> 68 #include <linux/mm.h> 69 #include <linux/string.h> 70 #include <linux/socket.h> 71 #include <linux/sockios.h> 72 #include <linux/errno.h> 73 #include <linux/in.h> 74 #include <linux/inet.h> 75 #include <linux/netdevice.h> 76 #include <linux/proc_fs.h> 77 #include <linux/init.h> 78 #include <linux/skbuff.h> 79 #include <linux/inetdevice.h> 80 #include <linux/igmp.h> 81 #include <linux/pkt_sched.h> 82 #include <linux/mroute.h> 83 #include <linux/netfilter_ipv4.h> 84 #include <linux/random.h> 85 #include <linux/rcupdate.h> 86 #include <linux/times.h> 87 #include <linux/slab.h> 88 #include <linux/jhash.h> 89 #include <net/dst.h> 90 #include <net/dst_metadata.h> 91 #include <net/net_namespace.h> 92 #include <net/protocol.h> 93 #include <net/ip.h> 94 #include <net/route.h> 95 #include <net/inetpeer.h> 96 #include <net/sock.h> 97 #include <net/ip_fib.h> 98 #include <net/nexthop.h> 99 #include <net/arp.h> 100 #include <net/tcp.h> 101 #include <net/icmp.h> 102 #include <net/xfrm.h> 103 #include <net/lwtunnel.h> 104 #include <net/netevent.h> 105 #include <net/rtnetlink.h> 106 #ifdef CONFIG_SYSCTL 107 #include <linux/sysctl.h> 108 #endif 109 #include <net/secure_seq.h> 110 #include <net/ip_tunnels.h> 111 #include <net/l3mdev.h> 112 113 #include "fib_lookup.h" 114 115 #define RT_FL_TOS(oldflp4) \ 116 ((oldflp4)->flowi4_tos & (IPTOS_RT_MASK | RTO_ONLINK)) 117 118 #define RT_GC_TIMEOUT (300*HZ) 119 120 static int ip_rt_max_size; 121 static int ip_rt_redirect_number __read_mostly = 9; 122 static int ip_rt_redirect_load __read_mostly = HZ / 50; 123 static int ip_rt_redirect_silence __read_mostly = ((HZ / 50) << (9 + 1)); 124 static int ip_rt_error_cost __read_mostly = HZ; 125 static int ip_rt_error_burst __read_mostly = 5 * HZ; 126 static int ip_rt_mtu_expires __read_mostly = 10 * 60 * HZ; 127 static u32 ip_rt_min_pmtu __read_mostly = 512 + 20 + 20; 128 static int ip_rt_min_advmss __read_mostly = 256; 129 130 static int ip_rt_gc_timeout __read_mostly = RT_GC_TIMEOUT; 131 132 /* 133 * Interface to generic destination cache. 134 */ 135 136 static struct dst_entry *ipv4_dst_check(struct dst_entry *dst, u32 cookie); 137 static unsigned int ipv4_default_advmss(const struct dst_entry *dst); 138 static unsigned int ipv4_mtu(const struct dst_entry *dst); 139 static struct dst_entry *ipv4_negative_advice(struct dst_entry *dst); 140 static void ipv4_link_failure(struct sk_buff *skb); 141 static void ip_rt_update_pmtu(struct dst_entry *dst, struct sock *sk, 142 struct sk_buff *skb, u32 mtu, 143 bool confirm_neigh); 144 static void ip_do_redirect(struct dst_entry *dst, struct sock *sk, 145 struct sk_buff *skb); 146 static void ipv4_dst_destroy(struct dst_entry *dst); 147 148 static u32 *ipv4_cow_metrics(struct dst_entry *dst, unsigned long old) 149 { 150 WARN_ON(1); 151 return NULL; 152 } 153 154 static struct neighbour *ipv4_neigh_lookup(const struct dst_entry *dst, 155 struct sk_buff *skb, 156 const void *daddr); 157 static void ipv4_confirm_neigh(const struct dst_entry *dst, const void *daddr); 158 159 static struct dst_ops ipv4_dst_ops = { 160 .family = AF_INET, 161 .check = ipv4_dst_check, 162 .default_advmss = ipv4_default_advmss, 163 .mtu = ipv4_mtu, 164 .cow_metrics = ipv4_cow_metrics, 165 .destroy = ipv4_dst_destroy, 166 .negative_advice = ipv4_negative_advice, 167 .link_failure = ipv4_link_failure, 168 .update_pmtu = ip_rt_update_pmtu, 169 .redirect = ip_do_redirect, 170 .local_out = __ip_local_out, 171 .neigh_lookup = ipv4_neigh_lookup, 172 .confirm_neigh = ipv4_confirm_neigh, 173 }; 174 175 #define ECN_OR_COST(class) TC_PRIO_##class 176 177 const __u8 ip_tos2prio[16] = { 178 TC_PRIO_BESTEFFORT, 179 ECN_OR_COST(BESTEFFORT), 180 TC_PRIO_BESTEFFORT, 181 ECN_OR_COST(BESTEFFORT), 182 TC_PRIO_BULK, 183 ECN_OR_COST(BULK), 184 TC_PRIO_BULK, 185 ECN_OR_COST(BULK), 186 TC_PRIO_INTERACTIVE, 187 ECN_OR_COST(INTERACTIVE), 188 TC_PRIO_INTERACTIVE, 189 ECN_OR_COST(INTERACTIVE), 190 TC_PRIO_INTERACTIVE_BULK, 191 ECN_OR_COST(INTERACTIVE_BULK), 192 TC_PRIO_INTERACTIVE_BULK, 193 ECN_OR_COST(INTERACTIVE_BULK) 194 }; 195 EXPORT_SYMBOL(ip_tos2prio); 196 197 static DEFINE_PER_CPU(struct rt_cache_stat, rt_cache_stat); 198 #define RT_CACHE_STAT_INC(field) raw_cpu_inc(rt_cache_stat.field) 199 200 #ifdef CONFIG_PROC_FS 201 static void *rt_cache_seq_start(struct seq_file *seq, loff_t *pos) 202 { 203 if (*pos) 204 return NULL; 205 return SEQ_START_TOKEN; 206 } 207 208 static void *rt_cache_seq_next(struct seq_file *seq, void *v, loff_t *pos) 209 { 210 ++*pos; 211 return NULL; 212 } 213 214 static void rt_cache_seq_stop(struct seq_file *seq, void *v) 215 { 216 } 217 218 static int rt_cache_seq_show(struct seq_file *seq, void *v) 219 { 220 if (v == SEQ_START_TOKEN) 221 seq_printf(seq, "%-127s\n", 222 "Iface\tDestination\tGateway \tFlags\t\tRefCnt\tUse\t" 223 "Metric\tSource\t\tMTU\tWindow\tIRTT\tTOS\tHHRef\t" 224 "HHUptod\tSpecDst"); 225 return 0; 226 } 227 228 static const struct seq_operations rt_cache_seq_ops = { 229 .start = rt_cache_seq_start, 230 .next = rt_cache_seq_next, 231 .stop = rt_cache_seq_stop, 232 .show = rt_cache_seq_show, 233 }; 234 235 static int rt_cache_seq_open(struct inode *inode, struct file *file) 236 { 237 return seq_open(file, &rt_cache_seq_ops); 238 } 239 240 static const struct proc_ops rt_cache_proc_ops = { 241 .proc_open = rt_cache_seq_open, 242 .proc_read = seq_read, 243 .proc_lseek = seq_lseek, 244 .proc_release = seq_release, 245 }; 246 247 248 static void *rt_cpu_seq_start(struct seq_file *seq, loff_t *pos) 249 { 250 int cpu; 251 252 if (*pos == 0) 253 return SEQ_START_TOKEN; 254 255 for (cpu = *pos-1; cpu < nr_cpu_ids; ++cpu) { 256 if (!cpu_possible(cpu)) 257 continue; 258 *pos = cpu+1; 259 return &per_cpu(rt_cache_stat, cpu); 260 } 261 return NULL; 262 } 263 264 static void *rt_cpu_seq_next(struct seq_file *seq, void *v, loff_t *pos) 265 { 266 int cpu; 267 268 for (cpu = *pos; cpu < nr_cpu_ids; ++cpu) { 269 if (!cpu_possible(cpu)) 270 continue; 271 *pos = cpu+1; 272 return &per_cpu(rt_cache_stat, cpu); 273 } 274 (*pos)++; 275 return NULL; 276 277 } 278 279 static void rt_cpu_seq_stop(struct seq_file *seq, void *v) 280 { 281 282 } 283 284 static int rt_cpu_seq_show(struct seq_file *seq, void *v) 285 { 286 struct rt_cache_stat *st = v; 287 288 if (v == SEQ_START_TOKEN) { 289 seq_printf(seq, "entries in_hit in_slow_tot in_slow_mc in_no_route in_brd in_martian_dst in_martian_src out_hit out_slow_tot out_slow_mc gc_total gc_ignored gc_goal_miss gc_dst_overflow in_hlist_search out_hlist_search\n"); 290 return 0; 291 } 292 293 seq_printf(seq,"%08x %08x %08x %08x %08x %08x %08x %08x " 294 " %08x %08x %08x %08x %08x %08x %08x %08x %08x \n", 295 dst_entries_get_slow(&ipv4_dst_ops), 296 0, /* st->in_hit */ 297 st->in_slow_tot, 298 st->in_slow_mc, 299 st->in_no_route, 300 st->in_brd, 301 st->in_martian_dst, 302 st->in_martian_src, 303 304 0, /* st->out_hit */ 305 st->out_slow_tot, 306 st->out_slow_mc, 307 308 0, /* st->gc_total */ 309 0, /* st->gc_ignored */ 310 0, /* st->gc_goal_miss */ 311 0, /* st->gc_dst_overflow */ 312 0, /* st->in_hlist_search */ 313 0 /* st->out_hlist_search */ 314 ); 315 return 0; 316 } 317 318 static const struct seq_operations rt_cpu_seq_ops = { 319 .start = rt_cpu_seq_start, 320 .next = rt_cpu_seq_next, 321 .stop = rt_cpu_seq_stop, 322 .show = rt_cpu_seq_show, 323 }; 324 325 326 static int rt_cpu_seq_open(struct inode *inode, struct file *file) 327 { 328 return seq_open(file, &rt_cpu_seq_ops); 329 } 330 331 static const struct proc_ops rt_cpu_proc_ops = { 332 .proc_open = rt_cpu_seq_open, 333 .proc_read = seq_read, 334 .proc_lseek = seq_lseek, 335 .proc_release = seq_release, 336 }; 337 338 #ifdef CONFIG_IP_ROUTE_CLASSID 339 static int rt_acct_proc_show(struct seq_file *m, void *v) 340 { 341 struct ip_rt_acct *dst, *src; 342 unsigned int i, j; 343 344 dst = kcalloc(256, sizeof(struct ip_rt_acct), GFP_KERNEL); 345 if (!dst) 346 return -ENOMEM; 347 348 for_each_possible_cpu(i) { 349 src = (struct ip_rt_acct *)per_cpu_ptr(ip_rt_acct, i); 350 for (j = 0; j < 256; j++) { 351 dst[j].o_bytes += src[j].o_bytes; 352 dst[j].o_packets += src[j].o_packets; 353 dst[j].i_bytes += src[j].i_bytes; 354 dst[j].i_packets += src[j].i_packets; 355 } 356 } 357 358 seq_write(m, dst, 256 * sizeof(struct ip_rt_acct)); 359 kfree(dst); 360 return 0; 361 } 362 #endif 363 364 static int __net_init ip_rt_do_proc_init(struct net *net) 365 { 366 struct proc_dir_entry *pde; 367 368 pde = proc_create("rt_cache", 0444, net->proc_net, 369 &rt_cache_proc_ops); 370 if (!pde) 371 goto err1; 372 373 pde = proc_create("rt_cache", 0444, 374 net->proc_net_stat, &rt_cpu_proc_ops); 375 if (!pde) 376 goto err2; 377 378 #ifdef CONFIG_IP_ROUTE_CLASSID 379 pde = proc_create_single("rt_acct", 0, net->proc_net, 380 rt_acct_proc_show); 381 if (!pde) 382 goto err3; 383 #endif 384 return 0; 385 386 #ifdef CONFIG_IP_ROUTE_CLASSID 387 err3: 388 remove_proc_entry("rt_cache", net->proc_net_stat); 389 #endif 390 err2: 391 remove_proc_entry("rt_cache", net->proc_net); 392 err1: 393 return -ENOMEM; 394 } 395 396 static void __net_exit ip_rt_do_proc_exit(struct net *net) 397 { 398 remove_proc_entry("rt_cache", net->proc_net_stat); 399 remove_proc_entry("rt_cache", net->proc_net); 400 #ifdef CONFIG_IP_ROUTE_CLASSID 401 remove_proc_entry("rt_acct", net->proc_net); 402 #endif 403 } 404 405 static struct pernet_operations ip_rt_proc_ops __net_initdata = { 406 .init = ip_rt_do_proc_init, 407 .exit = ip_rt_do_proc_exit, 408 }; 409 410 static int __init ip_rt_proc_init(void) 411 { 412 return register_pernet_subsys(&ip_rt_proc_ops); 413 } 414 415 #else 416 static inline int ip_rt_proc_init(void) 417 { 418 return 0; 419 } 420 #endif /* CONFIG_PROC_FS */ 421 422 static inline bool rt_is_expired(const struct rtable *rth) 423 { 424 return rth->rt_genid != rt_genid_ipv4(dev_net(rth->dst.dev)); 425 } 426 427 void rt_cache_flush(struct net *net) 428 { 429 rt_genid_bump_ipv4(net); 430 } 431 432 static struct neighbour *ipv4_neigh_lookup(const struct dst_entry *dst, 433 struct sk_buff *skb, 434 const void *daddr) 435 { 436 const struct rtable *rt = container_of(dst, struct rtable, dst); 437 struct net_device *dev = dst->dev; 438 struct neighbour *n; 439 440 rcu_read_lock_bh(); 441 442 if (likely(rt->rt_gw_family == AF_INET)) { 443 n = ip_neigh_gw4(dev, rt->rt_gw4); 444 } else if (rt->rt_gw_family == AF_INET6) { 445 n = ip_neigh_gw6(dev, &rt->rt_gw6); 446 } else { 447 __be32 pkey; 448 449 pkey = skb ? ip_hdr(skb)->daddr : *((__be32 *) daddr); 450 n = ip_neigh_gw4(dev, pkey); 451 } 452 453 if (!IS_ERR(n) && !refcount_inc_not_zero(&n->refcnt)) 454 n = NULL; 455 456 rcu_read_unlock_bh(); 457 458 return n; 459 } 460 461 static void ipv4_confirm_neigh(const struct dst_entry *dst, const void *daddr) 462 { 463 const struct rtable *rt = container_of(dst, struct rtable, dst); 464 struct net_device *dev = dst->dev; 465 const __be32 *pkey = daddr; 466 467 if (rt->rt_gw_family == AF_INET) { 468 pkey = (const __be32 *)&rt->rt_gw4; 469 } else if (rt->rt_gw_family == AF_INET6) { 470 return __ipv6_confirm_neigh_stub(dev, &rt->rt_gw6); 471 } else if (!daddr || 472 (rt->rt_flags & 473 (RTCF_MULTICAST | RTCF_BROADCAST | RTCF_LOCAL))) { 474 return; 475 } 476 __ipv4_confirm_neigh(dev, *(__force u32 *)pkey); 477 } 478 479 #define IP_IDENTS_SZ 2048u 480 481 static atomic_t *ip_idents __read_mostly; 482 static u32 *ip_tstamps __read_mostly; 483 484 /* In order to protect privacy, we add a perturbation to identifiers 485 * if one generator is seldom used. This makes hard for an attacker 486 * to infer how many packets were sent between two points in time. 487 */ 488 u32 ip_idents_reserve(u32 hash, int segs) 489 { 490 u32 *p_tstamp = ip_tstamps + hash % IP_IDENTS_SZ; 491 atomic_t *p_id = ip_idents + hash % IP_IDENTS_SZ; 492 u32 old = READ_ONCE(*p_tstamp); 493 u32 now = (u32)jiffies; 494 u32 delta = 0; 495 496 if (old != now && cmpxchg(p_tstamp, old, now) == old) 497 delta = prandom_u32_max(now - old); 498 499 /* If UBSAN reports an error there, please make sure your compiler 500 * supports -fno-strict-overflow before reporting it that was a bug 501 * in UBSAN, and it has been fixed in GCC-8. 502 */ 503 return atomic_add_return(segs + delta, p_id) - segs; 504 } 505 EXPORT_SYMBOL(ip_idents_reserve); 506 507 void __ip_select_ident(struct net *net, struct iphdr *iph, int segs) 508 { 509 u32 hash, id; 510 511 /* Note the following code is not safe, but this is okay. */ 512 if (unlikely(siphash_key_is_zero(&net->ipv4.ip_id_key))) 513 get_random_bytes(&net->ipv4.ip_id_key, 514 sizeof(net->ipv4.ip_id_key)); 515 516 hash = siphash_3u32((__force u32)iph->daddr, 517 (__force u32)iph->saddr, 518 iph->protocol, 519 &net->ipv4.ip_id_key); 520 id = ip_idents_reserve(hash, segs); 521 iph->id = htons(id); 522 } 523 EXPORT_SYMBOL(__ip_select_ident); 524 525 static void __build_flow_key(const struct net *net, struct flowi4 *fl4, 526 const struct sock *sk, 527 const struct iphdr *iph, 528 int oif, u8 tos, 529 u8 prot, u32 mark, int flow_flags) 530 { 531 if (sk) { 532 const struct inet_sock *inet = inet_sk(sk); 533 534 oif = sk->sk_bound_dev_if; 535 mark = sk->sk_mark; 536 tos = RT_CONN_FLAGS(sk); 537 prot = inet->hdrincl ? IPPROTO_RAW : sk->sk_protocol; 538 } 539 flowi4_init_output(fl4, oif, mark, tos, 540 RT_SCOPE_UNIVERSE, prot, 541 flow_flags, 542 iph->daddr, iph->saddr, 0, 0, 543 sock_net_uid(net, sk)); 544 } 545 546 static void build_skb_flow_key(struct flowi4 *fl4, const struct sk_buff *skb, 547 const struct sock *sk) 548 { 549 const struct net *net = dev_net(skb->dev); 550 const struct iphdr *iph = ip_hdr(skb); 551 int oif = skb->dev->ifindex; 552 u8 tos = RT_TOS(iph->tos); 553 u8 prot = iph->protocol; 554 u32 mark = skb->mark; 555 556 __build_flow_key(net, fl4, sk, iph, oif, tos, prot, mark, 0); 557 } 558 559 static void build_sk_flow_key(struct flowi4 *fl4, const struct sock *sk) 560 { 561 const struct inet_sock *inet = inet_sk(sk); 562 const struct ip_options_rcu *inet_opt; 563 __be32 daddr = inet->inet_daddr; 564 565 rcu_read_lock(); 566 inet_opt = rcu_dereference(inet->inet_opt); 567 if (inet_opt && inet_opt->opt.srr) 568 daddr = inet_opt->opt.faddr; 569 flowi4_init_output(fl4, sk->sk_bound_dev_if, sk->sk_mark, 570 RT_CONN_FLAGS(sk), RT_SCOPE_UNIVERSE, 571 inet->hdrincl ? IPPROTO_RAW : sk->sk_protocol, 572 inet_sk_flowi_flags(sk), 573 daddr, inet->inet_saddr, 0, 0, sk->sk_uid); 574 rcu_read_unlock(); 575 } 576 577 static void ip_rt_build_flow_key(struct flowi4 *fl4, const struct sock *sk, 578 const struct sk_buff *skb) 579 { 580 if (skb) 581 build_skb_flow_key(fl4, skb, sk); 582 else 583 build_sk_flow_key(fl4, sk); 584 } 585 586 static DEFINE_SPINLOCK(fnhe_lock); 587 588 static void fnhe_flush_routes(struct fib_nh_exception *fnhe) 589 { 590 struct rtable *rt; 591 592 rt = rcu_dereference(fnhe->fnhe_rth_input); 593 if (rt) { 594 RCU_INIT_POINTER(fnhe->fnhe_rth_input, NULL); 595 dst_dev_put(&rt->dst); 596 dst_release(&rt->dst); 597 } 598 rt = rcu_dereference(fnhe->fnhe_rth_output); 599 if (rt) { 600 RCU_INIT_POINTER(fnhe->fnhe_rth_output, NULL); 601 dst_dev_put(&rt->dst); 602 dst_release(&rt->dst); 603 } 604 } 605 606 static struct fib_nh_exception *fnhe_oldest(struct fnhe_hash_bucket *hash) 607 { 608 struct fib_nh_exception *fnhe, *oldest; 609 610 oldest = rcu_dereference(hash->chain); 611 for (fnhe = rcu_dereference(oldest->fnhe_next); fnhe; 612 fnhe = rcu_dereference(fnhe->fnhe_next)) { 613 if (time_before(fnhe->fnhe_stamp, oldest->fnhe_stamp)) 614 oldest = fnhe; 615 } 616 fnhe_flush_routes(oldest); 617 return oldest; 618 } 619 620 static inline u32 fnhe_hashfun(__be32 daddr) 621 { 622 static u32 fnhe_hashrnd __read_mostly; 623 u32 hval; 624 625 net_get_random_once(&fnhe_hashrnd, sizeof(fnhe_hashrnd)); 626 hval = jhash_1word((__force u32)daddr, fnhe_hashrnd); 627 return hash_32(hval, FNHE_HASH_SHIFT); 628 } 629 630 static void fill_route_from_fnhe(struct rtable *rt, struct fib_nh_exception *fnhe) 631 { 632 rt->rt_pmtu = fnhe->fnhe_pmtu; 633 rt->rt_mtu_locked = fnhe->fnhe_mtu_locked; 634 rt->dst.expires = fnhe->fnhe_expires; 635 636 if (fnhe->fnhe_gw) { 637 rt->rt_flags |= RTCF_REDIRECTED; 638 rt->rt_uses_gateway = 1; 639 rt->rt_gw_family = AF_INET; 640 rt->rt_gw4 = fnhe->fnhe_gw; 641 } 642 } 643 644 static void update_or_create_fnhe(struct fib_nh_common *nhc, __be32 daddr, 645 __be32 gw, u32 pmtu, bool lock, 646 unsigned long expires) 647 { 648 struct fnhe_hash_bucket *hash; 649 struct fib_nh_exception *fnhe; 650 struct rtable *rt; 651 u32 genid, hval; 652 unsigned int i; 653 int depth; 654 655 genid = fnhe_genid(dev_net(nhc->nhc_dev)); 656 hval = fnhe_hashfun(daddr); 657 658 spin_lock_bh(&fnhe_lock); 659 660 hash = rcu_dereference(nhc->nhc_exceptions); 661 if (!hash) { 662 hash = kcalloc(FNHE_HASH_SIZE, sizeof(*hash), GFP_ATOMIC); 663 if (!hash) 664 goto out_unlock; 665 rcu_assign_pointer(nhc->nhc_exceptions, hash); 666 } 667 668 hash += hval; 669 670 depth = 0; 671 for (fnhe = rcu_dereference(hash->chain); fnhe; 672 fnhe = rcu_dereference(fnhe->fnhe_next)) { 673 if (fnhe->fnhe_daddr == daddr) 674 break; 675 depth++; 676 } 677 678 if (fnhe) { 679 if (fnhe->fnhe_genid != genid) 680 fnhe->fnhe_genid = genid; 681 if (gw) 682 fnhe->fnhe_gw = gw; 683 if (pmtu) { 684 fnhe->fnhe_pmtu = pmtu; 685 fnhe->fnhe_mtu_locked = lock; 686 } 687 fnhe->fnhe_expires = max(1UL, expires); 688 /* Update all cached dsts too */ 689 rt = rcu_dereference(fnhe->fnhe_rth_input); 690 if (rt) 691 fill_route_from_fnhe(rt, fnhe); 692 rt = rcu_dereference(fnhe->fnhe_rth_output); 693 if (rt) 694 fill_route_from_fnhe(rt, fnhe); 695 } else { 696 if (depth > FNHE_RECLAIM_DEPTH) 697 fnhe = fnhe_oldest(hash); 698 else { 699 fnhe = kzalloc(sizeof(*fnhe), GFP_ATOMIC); 700 if (!fnhe) 701 goto out_unlock; 702 703 fnhe->fnhe_next = hash->chain; 704 rcu_assign_pointer(hash->chain, fnhe); 705 } 706 fnhe->fnhe_genid = genid; 707 fnhe->fnhe_daddr = daddr; 708 fnhe->fnhe_gw = gw; 709 fnhe->fnhe_pmtu = pmtu; 710 fnhe->fnhe_mtu_locked = lock; 711 fnhe->fnhe_expires = max(1UL, expires); 712 713 /* Exception created; mark the cached routes for the nexthop 714 * stale, so anyone caching it rechecks if this exception 715 * applies to them. 716 */ 717 rt = rcu_dereference(nhc->nhc_rth_input); 718 if (rt) 719 rt->dst.obsolete = DST_OBSOLETE_KILL; 720 721 for_each_possible_cpu(i) { 722 struct rtable __rcu **prt; 723 prt = per_cpu_ptr(nhc->nhc_pcpu_rth_output, i); 724 rt = rcu_dereference(*prt); 725 if (rt) 726 rt->dst.obsolete = DST_OBSOLETE_KILL; 727 } 728 } 729 730 fnhe->fnhe_stamp = jiffies; 731 732 out_unlock: 733 spin_unlock_bh(&fnhe_lock); 734 } 735 736 static void __ip_do_redirect(struct rtable *rt, struct sk_buff *skb, struct flowi4 *fl4, 737 bool kill_route) 738 { 739 __be32 new_gw = icmp_hdr(skb)->un.gateway; 740 __be32 old_gw = ip_hdr(skb)->saddr; 741 struct net_device *dev = skb->dev; 742 struct in_device *in_dev; 743 struct fib_result res; 744 struct neighbour *n; 745 struct net *net; 746 747 switch (icmp_hdr(skb)->code & 7) { 748 case ICMP_REDIR_NET: 749 case ICMP_REDIR_NETTOS: 750 case ICMP_REDIR_HOST: 751 case ICMP_REDIR_HOSTTOS: 752 break; 753 754 default: 755 return; 756 } 757 758 if (rt->rt_gw_family != AF_INET || rt->rt_gw4 != old_gw) 759 return; 760 761 in_dev = __in_dev_get_rcu(dev); 762 if (!in_dev) 763 return; 764 765 net = dev_net(dev); 766 if (new_gw == old_gw || !IN_DEV_RX_REDIRECTS(in_dev) || 767 ipv4_is_multicast(new_gw) || ipv4_is_lbcast(new_gw) || 768 ipv4_is_zeronet(new_gw)) 769 goto reject_redirect; 770 771 if (!IN_DEV_SHARED_MEDIA(in_dev)) { 772 if (!inet_addr_onlink(in_dev, new_gw, old_gw)) 773 goto reject_redirect; 774 if (IN_DEV_SEC_REDIRECTS(in_dev) && ip_fib_check_default(new_gw, dev)) 775 goto reject_redirect; 776 } else { 777 if (inet_addr_type(net, new_gw) != RTN_UNICAST) 778 goto reject_redirect; 779 } 780 781 n = __ipv4_neigh_lookup(rt->dst.dev, new_gw); 782 if (!n) 783 n = neigh_create(&arp_tbl, &new_gw, rt->dst.dev); 784 if (!IS_ERR(n)) { 785 if (!(n->nud_state & NUD_VALID)) { 786 neigh_event_send(n, NULL); 787 } else { 788 if (fib_lookup(net, fl4, &res, 0) == 0) { 789 struct fib_nh_common *nhc; 790 791 fib_select_path(net, &res, fl4, skb); 792 nhc = FIB_RES_NHC(res); 793 update_or_create_fnhe(nhc, fl4->daddr, new_gw, 794 0, false, 795 jiffies + ip_rt_gc_timeout); 796 } 797 if (kill_route) 798 rt->dst.obsolete = DST_OBSOLETE_KILL; 799 call_netevent_notifiers(NETEVENT_NEIGH_UPDATE, n); 800 } 801 neigh_release(n); 802 } 803 return; 804 805 reject_redirect: 806 #ifdef CONFIG_IP_ROUTE_VERBOSE 807 if (IN_DEV_LOG_MARTIANS(in_dev)) { 808 const struct iphdr *iph = (const struct iphdr *) skb->data; 809 __be32 daddr = iph->daddr; 810 __be32 saddr = iph->saddr; 811 812 net_info_ratelimited("Redirect from %pI4 on %s about %pI4 ignored\n" 813 " Advised path = %pI4 -> %pI4\n", 814 &old_gw, dev->name, &new_gw, 815 &saddr, &daddr); 816 } 817 #endif 818 ; 819 } 820 821 static void ip_do_redirect(struct dst_entry *dst, struct sock *sk, struct sk_buff *skb) 822 { 823 struct rtable *rt; 824 struct flowi4 fl4; 825 const struct iphdr *iph = (const struct iphdr *) skb->data; 826 struct net *net = dev_net(skb->dev); 827 int oif = skb->dev->ifindex; 828 u8 tos = RT_TOS(iph->tos); 829 u8 prot = iph->protocol; 830 u32 mark = skb->mark; 831 832 rt = (struct rtable *) dst; 833 834 __build_flow_key(net, &fl4, sk, iph, oif, tos, prot, mark, 0); 835 __ip_do_redirect(rt, skb, &fl4, true); 836 } 837 838 static struct dst_entry *ipv4_negative_advice(struct dst_entry *dst) 839 { 840 struct rtable *rt = (struct rtable *)dst; 841 struct dst_entry *ret = dst; 842 843 if (rt) { 844 if (dst->obsolete > 0) { 845 ip_rt_put(rt); 846 ret = NULL; 847 } else if ((rt->rt_flags & RTCF_REDIRECTED) || 848 rt->dst.expires) { 849 ip_rt_put(rt); 850 ret = NULL; 851 } 852 } 853 return ret; 854 } 855 856 /* 857 * Algorithm: 858 * 1. The first ip_rt_redirect_number redirects are sent 859 * with exponential backoff, then we stop sending them at all, 860 * assuming that the host ignores our redirects. 861 * 2. If we did not see packets requiring redirects 862 * during ip_rt_redirect_silence, we assume that the host 863 * forgot redirected route and start to send redirects again. 864 * 865 * This algorithm is much cheaper and more intelligent than dumb load limiting 866 * in icmp.c. 867 * 868 * NOTE. Do not forget to inhibit load limiting for redirects (redundant) 869 * and "frag. need" (breaks PMTU discovery) in icmp.c. 870 */ 871 872 void ip_rt_send_redirect(struct sk_buff *skb) 873 { 874 struct rtable *rt = skb_rtable(skb); 875 struct in_device *in_dev; 876 struct inet_peer *peer; 877 struct net *net; 878 int log_martians; 879 int vif; 880 881 rcu_read_lock(); 882 in_dev = __in_dev_get_rcu(rt->dst.dev); 883 if (!in_dev || !IN_DEV_TX_REDIRECTS(in_dev)) { 884 rcu_read_unlock(); 885 return; 886 } 887 log_martians = IN_DEV_LOG_MARTIANS(in_dev); 888 vif = l3mdev_master_ifindex_rcu(rt->dst.dev); 889 rcu_read_unlock(); 890 891 net = dev_net(rt->dst.dev); 892 peer = inet_getpeer_v4(net->ipv4.peers, ip_hdr(skb)->saddr, vif, 1); 893 if (!peer) { 894 icmp_send(skb, ICMP_REDIRECT, ICMP_REDIR_HOST, 895 rt_nexthop(rt, ip_hdr(skb)->daddr)); 896 return; 897 } 898 899 /* No redirected packets during ip_rt_redirect_silence; 900 * reset the algorithm. 901 */ 902 if (time_after(jiffies, peer->rate_last + ip_rt_redirect_silence)) { 903 peer->rate_tokens = 0; 904 peer->n_redirects = 0; 905 } 906 907 /* Too many ignored redirects; do not send anything 908 * set dst.rate_last to the last seen redirected packet. 909 */ 910 if (peer->n_redirects >= ip_rt_redirect_number) { 911 peer->rate_last = jiffies; 912 goto out_put_peer; 913 } 914 915 /* Check for load limit; set rate_last to the latest sent 916 * redirect. 917 */ 918 if (peer->n_redirects == 0 || 919 time_after(jiffies, 920 (peer->rate_last + 921 (ip_rt_redirect_load << peer->n_redirects)))) { 922 __be32 gw = rt_nexthop(rt, ip_hdr(skb)->daddr); 923 924 icmp_send(skb, ICMP_REDIRECT, ICMP_REDIR_HOST, gw); 925 peer->rate_last = jiffies; 926 ++peer->n_redirects; 927 #ifdef CONFIG_IP_ROUTE_VERBOSE 928 if (log_martians && 929 peer->n_redirects == ip_rt_redirect_number) 930 net_warn_ratelimited("host %pI4/if%d ignores redirects for %pI4 to %pI4\n", 931 &ip_hdr(skb)->saddr, inet_iif(skb), 932 &ip_hdr(skb)->daddr, &gw); 933 #endif 934 } 935 out_put_peer: 936 inet_putpeer(peer); 937 } 938 939 static int ip_error(struct sk_buff *skb) 940 { 941 struct rtable *rt = skb_rtable(skb); 942 struct net_device *dev = skb->dev; 943 struct in_device *in_dev; 944 struct inet_peer *peer; 945 unsigned long now; 946 struct net *net; 947 bool send; 948 int code; 949 950 if (netif_is_l3_master(skb->dev)) { 951 dev = __dev_get_by_index(dev_net(skb->dev), IPCB(skb)->iif); 952 if (!dev) 953 goto out; 954 } 955 956 in_dev = __in_dev_get_rcu(dev); 957 958 /* IP on this device is disabled. */ 959 if (!in_dev) 960 goto out; 961 962 net = dev_net(rt->dst.dev); 963 if (!IN_DEV_FORWARD(in_dev)) { 964 switch (rt->dst.error) { 965 case EHOSTUNREACH: 966 __IP_INC_STATS(net, IPSTATS_MIB_INADDRERRORS); 967 break; 968 969 case ENETUNREACH: 970 __IP_INC_STATS(net, IPSTATS_MIB_INNOROUTES); 971 break; 972 } 973 goto out; 974 } 975 976 switch (rt->dst.error) { 977 case EINVAL: 978 default: 979 goto out; 980 case EHOSTUNREACH: 981 code = ICMP_HOST_UNREACH; 982 break; 983 case ENETUNREACH: 984 code = ICMP_NET_UNREACH; 985 __IP_INC_STATS(net, IPSTATS_MIB_INNOROUTES); 986 break; 987 case EACCES: 988 code = ICMP_PKT_FILTERED; 989 break; 990 } 991 992 peer = inet_getpeer_v4(net->ipv4.peers, ip_hdr(skb)->saddr, 993 l3mdev_master_ifindex(skb->dev), 1); 994 995 send = true; 996 if (peer) { 997 now = jiffies; 998 peer->rate_tokens += now - peer->rate_last; 999 if (peer->rate_tokens > ip_rt_error_burst) 1000 peer->rate_tokens = ip_rt_error_burst; 1001 peer->rate_last = now; 1002 if (peer->rate_tokens >= ip_rt_error_cost) 1003 peer->rate_tokens -= ip_rt_error_cost; 1004 else 1005 send = false; 1006 inet_putpeer(peer); 1007 } 1008 if (send) 1009 icmp_send(skb, ICMP_DEST_UNREACH, code, 0); 1010 1011 out: kfree_skb(skb); 1012 return 0; 1013 } 1014 1015 static void __ip_rt_update_pmtu(struct rtable *rt, struct flowi4 *fl4, u32 mtu) 1016 { 1017 struct dst_entry *dst = &rt->dst; 1018 struct net *net = dev_net(dst->dev); 1019 struct fib_result res; 1020 bool lock = false; 1021 u32 old_mtu; 1022 1023 if (ip_mtu_locked(dst)) 1024 return; 1025 1026 old_mtu = ipv4_mtu(dst); 1027 if (old_mtu < mtu) 1028 return; 1029 1030 if (mtu < ip_rt_min_pmtu) { 1031 lock = true; 1032 mtu = min(old_mtu, ip_rt_min_pmtu); 1033 } 1034 1035 if (rt->rt_pmtu == mtu && !lock && 1036 time_before(jiffies, dst->expires - ip_rt_mtu_expires / 2)) 1037 return; 1038 1039 rcu_read_lock(); 1040 if (fib_lookup(net, fl4, &res, 0) == 0) { 1041 struct fib_nh_common *nhc; 1042 1043 fib_select_path(net, &res, fl4, NULL); 1044 nhc = FIB_RES_NHC(res); 1045 update_or_create_fnhe(nhc, fl4->daddr, 0, mtu, lock, 1046 jiffies + ip_rt_mtu_expires); 1047 } 1048 rcu_read_unlock(); 1049 } 1050 1051 static void ip_rt_update_pmtu(struct dst_entry *dst, struct sock *sk, 1052 struct sk_buff *skb, u32 mtu, 1053 bool confirm_neigh) 1054 { 1055 struct rtable *rt = (struct rtable *) dst; 1056 struct flowi4 fl4; 1057 1058 ip_rt_build_flow_key(&fl4, sk, skb); 1059 1060 /* Don't make lookup fail for bridged encapsulations */ 1061 if (skb && netif_is_any_bridge_port(skb->dev)) 1062 fl4.flowi4_oif = 0; 1063 1064 __ip_rt_update_pmtu(rt, &fl4, mtu); 1065 } 1066 1067 void ipv4_update_pmtu(struct sk_buff *skb, struct net *net, u32 mtu, 1068 int oif, u8 protocol) 1069 { 1070 const struct iphdr *iph = (const struct iphdr *)skb->data; 1071 struct flowi4 fl4; 1072 struct rtable *rt; 1073 u32 mark = IP4_REPLY_MARK(net, skb->mark); 1074 1075 __build_flow_key(net, &fl4, NULL, iph, oif, 1076 RT_TOS(iph->tos), protocol, mark, 0); 1077 rt = __ip_route_output_key(net, &fl4); 1078 if (!IS_ERR(rt)) { 1079 __ip_rt_update_pmtu(rt, &fl4, mtu); 1080 ip_rt_put(rt); 1081 } 1082 } 1083 EXPORT_SYMBOL_GPL(ipv4_update_pmtu); 1084 1085 static void __ipv4_sk_update_pmtu(struct sk_buff *skb, struct sock *sk, u32 mtu) 1086 { 1087 const struct iphdr *iph = (const struct iphdr *)skb->data; 1088 struct flowi4 fl4; 1089 struct rtable *rt; 1090 1091 __build_flow_key(sock_net(sk), &fl4, sk, iph, 0, 0, 0, 0, 0); 1092 1093 if (!fl4.flowi4_mark) 1094 fl4.flowi4_mark = IP4_REPLY_MARK(sock_net(sk), skb->mark); 1095 1096 rt = __ip_route_output_key(sock_net(sk), &fl4); 1097 if (!IS_ERR(rt)) { 1098 __ip_rt_update_pmtu(rt, &fl4, mtu); 1099 ip_rt_put(rt); 1100 } 1101 } 1102 1103 void ipv4_sk_update_pmtu(struct sk_buff *skb, struct sock *sk, u32 mtu) 1104 { 1105 const struct iphdr *iph = (const struct iphdr *)skb->data; 1106 struct flowi4 fl4; 1107 struct rtable *rt; 1108 struct dst_entry *odst = NULL; 1109 bool new = false; 1110 struct net *net = sock_net(sk); 1111 1112 bh_lock_sock(sk); 1113 1114 if (!ip_sk_accept_pmtu(sk)) 1115 goto out; 1116 1117 odst = sk_dst_get(sk); 1118 1119 if (sock_owned_by_user(sk) || !odst) { 1120 __ipv4_sk_update_pmtu(skb, sk, mtu); 1121 goto out; 1122 } 1123 1124 __build_flow_key(net, &fl4, sk, iph, 0, 0, 0, 0, 0); 1125 1126 rt = (struct rtable *)odst; 1127 if (odst->obsolete && !odst->ops->check(odst, 0)) { 1128 rt = ip_route_output_flow(sock_net(sk), &fl4, sk); 1129 if (IS_ERR(rt)) 1130 goto out; 1131 1132 new = true; 1133 } 1134 1135 __ip_rt_update_pmtu((struct rtable *)xfrm_dst_path(&rt->dst), &fl4, mtu); 1136 1137 if (!dst_check(&rt->dst, 0)) { 1138 if (new) 1139 dst_release(&rt->dst); 1140 1141 rt = ip_route_output_flow(sock_net(sk), &fl4, sk); 1142 if (IS_ERR(rt)) 1143 goto out; 1144 1145 new = true; 1146 } 1147 1148 if (new) 1149 sk_dst_set(sk, &rt->dst); 1150 1151 out: 1152 bh_unlock_sock(sk); 1153 dst_release(odst); 1154 } 1155 EXPORT_SYMBOL_GPL(ipv4_sk_update_pmtu); 1156 1157 void ipv4_redirect(struct sk_buff *skb, struct net *net, 1158 int oif, u8 protocol) 1159 { 1160 const struct iphdr *iph = (const struct iphdr *)skb->data; 1161 struct flowi4 fl4; 1162 struct rtable *rt; 1163 1164 __build_flow_key(net, &fl4, NULL, iph, oif, 1165 RT_TOS(iph->tos), protocol, 0, 0); 1166 rt = __ip_route_output_key(net, &fl4); 1167 if (!IS_ERR(rt)) { 1168 __ip_do_redirect(rt, skb, &fl4, false); 1169 ip_rt_put(rt); 1170 } 1171 } 1172 EXPORT_SYMBOL_GPL(ipv4_redirect); 1173 1174 void ipv4_sk_redirect(struct sk_buff *skb, struct sock *sk) 1175 { 1176 const struct iphdr *iph = (const struct iphdr *)skb->data; 1177 struct flowi4 fl4; 1178 struct rtable *rt; 1179 struct net *net = sock_net(sk); 1180 1181 __build_flow_key(net, &fl4, sk, iph, 0, 0, 0, 0, 0); 1182 rt = __ip_route_output_key(net, &fl4); 1183 if (!IS_ERR(rt)) { 1184 __ip_do_redirect(rt, skb, &fl4, false); 1185 ip_rt_put(rt); 1186 } 1187 } 1188 EXPORT_SYMBOL_GPL(ipv4_sk_redirect); 1189 1190 static struct dst_entry *ipv4_dst_check(struct dst_entry *dst, u32 cookie) 1191 { 1192 struct rtable *rt = (struct rtable *) dst; 1193 1194 /* All IPV4 dsts are created with ->obsolete set to the value 1195 * DST_OBSOLETE_FORCE_CHK which forces validation calls down 1196 * into this function always. 1197 * 1198 * When a PMTU/redirect information update invalidates a route, 1199 * this is indicated by setting obsolete to DST_OBSOLETE_KILL or 1200 * DST_OBSOLETE_DEAD. 1201 */ 1202 if (dst->obsolete != DST_OBSOLETE_FORCE_CHK || rt_is_expired(rt)) 1203 return NULL; 1204 return dst; 1205 } 1206 1207 static void ipv4_send_dest_unreach(struct sk_buff *skb) 1208 { 1209 struct ip_options opt; 1210 int res; 1211 1212 /* Recompile ip options since IPCB may not be valid anymore. 1213 * Also check we have a reasonable ipv4 header. 1214 */ 1215 if (!pskb_network_may_pull(skb, sizeof(struct iphdr)) || 1216 ip_hdr(skb)->version != 4 || ip_hdr(skb)->ihl < 5) 1217 return; 1218 1219 memset(&opt, 0, sizeof(opt)); 1220 if (ip_hdr(skb)->ihl > 5) { 1221 if (!pskb_network_may_pull(skb, ip_hdr(skb)->ihl * 4)) 1222 return; 1223 opt.optlen = ip_hdr(skb)->ihl * 4 - sizeof(struct iphdr); 1224 1225 rcu_read_lock(); 1226 res = __ip_options_compile(dev_net(skb->dev), &opt, skb, NULL); 1227 rcu_read_unlock(); 1228 1229 if (res) 1230 return; 1231 } 1232 __icmp_send(skb, ICMP_DEST_UNREACH, ICMP_HOST_UNREACH, 0, &opt); 1233 } 1234 1235 static void ipv4_link_failure(struct sk_buff *skb) 1236 { 1237 struct rtable *rt; 1238 1239 ipv4_send_dest_unreach(skb); 1240 1241 rt = skb_rtable(skb); 1242 if (rt) 1243 dst_set_expires(&rt->dst, 0); 1244 } 1245 1246 static int ip_rt_bug(struct net *net, struct sock *sk, struct sk_buff *skb) 1247 { 1248 pr_debug("%s: %pI4 -> %pI4, %s\n", 1249 __func__, &ip_hdr(skb)->saddr, &ip_hdr(skb)->daddr, 1250 skb->dev ? skb->dev->name : "?"); 1251 kfree_skb(skb); 1252 WARN_ON(1); 1253 return 0; 1254 } 1255 1256 /* 1257 We do not cache source address of outgoing interface, 1258 because it is used only by IP RR, TS and SRR options, 1259 so that it out of fast path. 1260 1261 BTW remember: "addr" is allowed to be not aligned 1262 in IP options! 1263 */ 1264 1265 void ip_rt_get_source(u8 *addr, struct sk_buff *skb, struct rtable *rt) 1266 { 1267 __be32 src; 1268 1269 if (rt_is_output_route(rt)) 1270 src = ip_hdr(skb)->saddr; 1271 else { 1272 struct fib_result res; 1273 struct iphdr *iph = ip_hdr(skb); 1274 struct flowi4 fl4 = { 1275 .daddr = iph->daddr, 1276 .saddr = iph->saddr, 1277 .flowi4_tos = RT_TOS(iph->tos), 1278 .flowi4_oif = rt->dst.dev->ifindex, 1279 .flowi4_iif = skb->dev->ifindex, 1280 .flowi4_mark = skb->mark, 1281 }; 1282 1283 rcu_read_lock(); 1284 if (fib_lookup(dev_net(rt->dst.dev), &fl4, &res, 0) == 0) 1285 src = fib_result_prefsrc(dev_net(rt->dst.dev), &res); 1286 else 1287 src = inet_select_addr(rt->dst.dev, 1288 rt_nexthop(rt, iph->daddr), 1289 RT_SCOPE_UNIVERSE); 1290 rcu_read_unlock(); 1291 } 1292 memcpy(addr, &src, 4); 1293 } 1294 1295 #ifdef CONFIG_IP_ROUTE_CLASSID 1296 static void set_class_tag(struct rtable *rt, u32 tag) 1297 { 1298 if (!(rt->dst.tclassid & 0xFFFF)) 1299 rt->dst.tclassid |= tag & 0xFFFF; 1300 if (!(rt->dst.tclassid & 0xFFFF0000)) 1301 rt->dst.tclassid |= tag & 0xFFFF0000; 1302 } 1303 #endif 1304 1305 static unsigned int ipv4_default_advmss(const struct dst_entry *dst) 1306 { 1307 unsigned int header_size = sizeof(struct tcphdr) + sizeof(struct iphdr); 1308 unsigned int advmss = max_t(unsigned int, ipv4_mtu(dst) - header_size, 1309 ip_rt_min_advmss); 1310 1311 return min(advmss, IPV4_MAX_PMTU - header_size); 1312 } 1313 1314 static unsigned int ipv4_mtu(const struct dst_entry *dst) 1315 { 1316 const struct rtable *rt = (const struct rtable *)dst; 1317 unsigned int mtu = rt->rt_pmtu; 1318 1319 if (!mtu || time_after_eq(jiffies, rt->dst.expires)) 1320 mtu = dst_metric_raw(dst, RTAX_MTU); 1321 1322 if (mtu) 1323 return mtu; 1324 1325 mtu = READ_ONCE(dst->dev->mtu); 1326 1327 if (unlikely(ip_mtu_locked(dst))) { 1328 if (rt->rt_uses_gateway && mtu > 576) 1329 mtu = 576; 1330 } 1331 1332 mtu = min_t(unsigned int, mtu, IP_MAX_MTU); 1333 1334 return mtu - lwtunnel_headroom(dst->lwtstate, mtu); 1335 } 1336 1337 static void ip_del_fnhe(struct fib_nh_common *nhc, __be32 daddr) 1338 { 1339 struct fnhe_hash_bucket *hash; 1340 struct fib_nh_exception *fnhe, __rcu **fnhe_p; 1341 u32 hval = fnhe_hashfun(daddr); 1342 1343 spin_lock_bh(&fnhe_lock); 1344 1345 hash = rcu_dereference_protected(nhc->nhc_exceptions, 1346 lockdep_is_held(&fnhe_lock)); 1347 hash += hval; 1348 1349 fnhe_p = &hash->chain; 1350 fnhe = rcu_dereference_protected(*fnhe_p, lockdep_is_held(&fnhe_lock)); 1351 while (fnhe) { 1352 if (fnhe->fnhe_daddr == daddr) { 1353 rcu_assign_pointer(*fnhe_p, rcu_dereference_protected( 1354 fnhe->fnhe_next, lockdep_is_held(&fnhe_lock))); 1355 /* set fnhe_daddr to 0 to ensure it won't bind with 1356 * new dsts in rt_bind_exception(). 1357 */ 1358 fnhe->fnhe_daddr = 0; 1359 fnhe_flush_routes(fnhe); 1360 kfree_rcu(fnhe, rcu); 1361 break; 1362 } 1363 fnhe_p = &fnhe->fnhe_next; 1364 fnhe = rcu_dereference_protected(fnhe->fnhe_next, 1365 lockdep_is_held(&fnhe_lock)); 1366 } 1367 1368 spin_unlock_bh(&fnhe_lock); 1369 } 1370 1371 static struct fib_nh_exception *find_exception(struct fib_nh_common *nhc, 1372 __be32 daddr) 1373 { 1374 struct fnhe_hash_bucket *hash = rcu_dereference(nhc->nhc_exceptions); 1375 struct fib_nh_exception *fnhe; 1376 u32 hval; 1377 1378 if (!hash) 1379 return NULL; 1380 1381 hval = fnhe_hashfun(daddr); 1382 1383 for (fnhe = rcu_dereference(hash[hval].chain); fnhe; 1384 fnhe = rcu_dereference(fnhe->fnhe_next)) { 1385 if (fnhe->fnhe_daddr == daddr) { 1386 if (fnhe->fnhe_expires && 1387 time_after(jiffies, fnhe->fnhe_expires)) { 1388 ip_del_fnhe(nhc, daddr); 1389 break; 1390 } 1391 return fnhe; 1392 } 1393 } 1394 return NULL; 1395 } 1396 1397 /* MTU selection: 1398 * 1. mtu on route is locked - use it 1399 * 2. mtu from nexthop exception 1400 * 3. mtu from egress device 1401 */ 1402 1403 u32 ip_mtu_from_fib_result(struct fib_result *res, __be32 daddr) 1404 { 1405 struct fib_nh_common *nhc = res->nhc; 1406 struct net_device *dev = nhc->nhc_dev; 1407 struct fib_info *fi = res->fi; 1408 u32 mtu = 0; 1409 1410 if (dev_net(dev)->ipv4.sysctl_ip_fwd_use_pmtu || 1411 fi->fib_metrics->metrics[RTAX_LOCK - 1] & (1 << RTAX_MTU)) 1412 mtu = fi->fib_mtu; 1413 1414 if (likely(!mtu)) { 1415 struct fib_nh_exception *fnhe; 1416 1417 fnhe = find_exception(nhc, daddr); 1418 if (fnhe && !time_after_eq(jiffies, fnhe->fnhe_expires)) 1419 mtu = fnhe->fnhe_pmtu; 1420 } 1421 1422 if (likely(!mtu)) 1423 mtu = min(READ_ONCE(dev->mtu), IP_MAX_MTU); 1424 1425 return mtu - lwtunnel_headroom(nhc->nhc_lwtstate, mtu); 1426 } 1427 1428 static bool rt_bind_exception(struct rtable *rt, struct fib_nh_exception *fnhe, 1429 __be32 daddr, const bool do_cache) 1430 { 1431 bool ret = false; 1432 1433 spin_lock_bh(&fnhe_lock); 1434 1435 if (daddr == fnhe->fnhe_daddr) { 1436 struct rtable __rcu **porig; 1437 struct rtable *orig; 1438 int genid = fnhe_genid(dev_net(rt->dst.dev)); 1439 1440 if (rt_is_input_route(rt)) 1441 porig = &fnhe->fnhe_rth_input; 1442 else 1443 porig = &fnhe->fnhe_rth_output; 1444 orig = rcu_dereference(*porig); 1445 1446 if (fnhe->fnhe_genid != genid) { 1447 fnhe->fnhe_genid = genid; 1448 fnhe->fnhe_gw = 0; 1449 fnhe->fnhe_pmtu = 0; 1450 fnhe->fnhe_expires = 0; 1451 fnhe->fnhe_mtu_locked = false; 1452 fnhe_flush_routes(fnhe); 1453 orig = NULL; 1454 } 1455 fill_route_from_fnhe(rt, fnhe); 1456 if (!rt->rt_gw4) { 1457 rt->rt_gw4 = daddr; 1458 rt->rt_gw_family = AF_INET; 1459 } 1460 1461 if (do_cache) { 1462 dst_hold(&rt->dst); 1463 rcu_assign_pointer(*porig, rt); 1464 if (orig) { 1465 dst_dev_put(&orig->dst); 1466 dst_release(&orig->dst); 1467 } 1468 ret = true; 1469 } 1470 1471 fnhe->fnhe_stamp = jiffies; 1472 } 1473 spin_unlock_bh(&fnhe_lock); 1474 1475 return ret; 1476 } 1477 1478 static bool rt_cache_route(struct fib_nh_common *nhc, struct rtable *rt) 1479 { 1480 struct rtable *orig, *prev, **p; 1481 bool ret = true; 1482 1483 if (rt_is_input_route(rt)) { 1484 p = (struct rtable **)&nhc->nhc_rth_input; 1485 } else { 1486 p = (struct rtable **)raw_cpu_ptr(nhc->nhc_pcpu_rth_output); 1487 } 1488 orig = *p; 1489 1490 /* hold dst before doing cmpxchg() to avoid race condition 1491 * on this dst 1492 */ 1493 dst_hold(&rt->dst); 1494 prev = cmpxchg(p, orig, rt); 1495 if (prev == orig) { 1496 if (orig) { 1497 rt_add_uncached_list(orig); 1498 dst_release(&orig->dst); 1499 } 1500 } else { 1501 dst_release(&rt->dst); 1502 ret = false; 1503 } 1504 1505 return ret; 1506 } 1507 1508 struct uncached_list { 1509 spinlock_t lock; 1510 struct list_head head; 1511 }; 1512 1513 static DEFINE_PER_CPU_ALIGNED(struct uncached_list, rt_uncached_list); 1514 1515 void rt_add_uncached_list(struct rtable *rt) 1516 { 1517 struct uncached_list *ul = raw_cpu_ptr(&rt_uncached_list); 1518 1519 rt->rt_uncached_list = ul; 1520 1521 spin_lock_bh(&ul->lock); 1522 list_add_tail(&rt->rt_uncached, &ul->head); 1523 spin_unlock_bh(&ul->lock); 1524 } 1525 1526 void rt_del_uncached_list(struct rtable *rt) 1527 { 1528 if (!list_empty(&rt->rt_uncached)) { 1529 struct uncached_list *ul = rt->rt_uncached_list; 1530 1531 spin_lock_bh(&ul->lock); 1532 list_del(&rt->rt_uncached); 1533 spin_unlock_bh(&ul->lock); 1534 } 1535 } 1536 1537 static void ipv4_dst_destroy(struct dst_entry *dst) 1538 { 1539 struct rtable *rt = (struct rtable *)dst; 1540 1541 ip_dst_metrics_put(dst); 1542 rt_del_uncached_list(rt); 1543 } 1544 1545 void rt_flush_dev(struct net_device *dev) 1546 { 1547 struct rtable *rt; 1548 int cpu; 1549 1550 for_each_possible_cpu(cpu) { 1551 struct uncached_list *ul = &per_cpu(rt_uncached_list, cpu); 1552 1553 spin_lock_bh(&ul->lock); 1554 list_for_each_entry(rt, &ul->head, rt_uncached) { 1555 if (rt->dst.dev != dev) 1556 continue; 1557 rt->dst.dev = blackhole_netdev; 1558 dev_hold(rt->dst.dev); 1559 dev_put(dev); 1560 } 1561 spin_unlock_bh(&ul->lock); 1562 } 1563 } 1564 1565 static bool rt_cache_valid(const struct rtable *rt) 1566 { 1567 return rt && 1568 rt->dst.obsolete == DST_OBSOLETE_FORCE_CHK && 1569 !rt_is_expired(rt); 1570 } 1571 1572 static void rt_set_nexthop(struct rtable *rt, __be32 daddr, 1573 const struct fib_result *res, 1574 struct fib_nh_exception *fnhe, 1575 struct fib_info *fi, u16 type, u32 itag, 1576 const bool do_cache) 1577 { 1578 bool cached = false; 1579 1580 if (fi) { 1581 struct fib_nh_common *nhc = FIB_RES_NHC(*res); 1582 1583 if (nhc->nhc_gw_family && nhc->nhc_scope == RT_SCOPE_LINK) { 1584 rt->rt_uses_gateway = 1; 1585 rt->rt_gw_family = nhc->nhc_gw_family; 1586 /* only INET and INET6 are supported */ 1587 if (likely(nhc->nhc_gw_family == AF_INET)) 1588 rt->rt_gw4 = nhc->nhc_gw.ipv4; 1589 else 1590 rt->rt_gw6 = nhc->nhc_gw.ipv6; 1591 } 1592 1593 ip_dst_init_metrics(&rt->dst, fi->fib_metrics); 1594 1595 #ifdef CONFIG_IP_ROUTE_CLASSID 1596 if (nhc->nhc_family == AF_INET) { 1597 struct fib_nh *nh; 1598 1599 nh = container_of(nhc, struct fib_nh, nh_common); 1600 rt->dst.tclassid = nh->nh_tclassid; 1601 } 1602 #endif 1603 rt->dst.lwtstate = lwtstate_get(nhc->nhc_lwtstate); 1604 if (unlikely(fnhe)) 1605 cached = rt_bind_exception(rt, fnhe, daddr, do_cache); 1606 else if (do_cache) 1607 cached = rt_cache_route(nhc, rt); 1608 if (unlikely(!cached)) { 1609 /* Routes we intend to cache in nexthop exception or 1610 * FIB nexthop have the DST_NOCACHE bit clear. 1611 * However, if we are unsuccessful at storing this 1612 * route into the cache we really need to set it. 1613 */ 1614 if (!rt->rt_gw4) { 1615 rt->rt_gw_family = AF_INET; 1616 rt->rt_gw4 = daddr; 1617 } 1618 rt_add_uncached_list(rt); 1619 } 1620 } else 1621 rt_add_uncached_list(rt); 1622 1623 #ifdef CONFIG_IP_ROUTE_CLASSID 1624 #ifdef CONFIG_IP_MULTIPLE_TABLES 1625 set_class_tag(rt, res->tclassid); 1626 #endif 1627 set_class_tag(rt, itag); 1628 #endif 1629 } 1630 1631 struct rtable *rt_dst_alloc(struct net_device *dev, 1632 unsigned int flags, u16 type, 1633 bool nopolicy, bool noxfrm) 1634 { 1635 struct rtable *rt; 1636 1637 rt = dst_alloc(&ipv4_dst_ops, dev, 1, DST_OBSOLETE_FORCE_CHK, 1638 (nopolicy ? DST_NOPOLICY : 0) | 1639 (noxfrm ? DST_NOXFRM : 0)); 1640 1641 if (rt) { 1642 rt->rt_genid = rt_genid_ipv4(dev_net(dev)); 1643 rt->rt_flags = flags; 1644 rt->rt_type = type; 1645 rt->rt_is_input = 0; 1646 rt->rt_iif = 0; 1647 rt->rt_pmtu = 0; 1648 rt->rt_mtu_locked = 0; 1649 rt->rt_uses_gateway = 0; 1650 rt->rt_gw_family = 0; 1651 rt->rt_gw4 = 0; 1652 INIT_LIST_HEAD(&rt->rt_uncached); 1653 1654 rt->dst.output = ip_output; 1655 if (flags & RTCF_LOCAL) 1656 rt->dst.input = ip_local_deliver; 1657 } 1658 1659 return rt; 1660 } 1661 EXPORT_SYMBOL(rt_dst_alloc); 1662 1663 struct rtable *rt_dst_clone(struct net_device *dev, struct rtable *rt) 1664 { 1665 struct rtable *new_rt; 1666 1667 new_rt = dst_alloc(&ipv4_dst_ops, dev, 1, DST_OBSOLETE_FORCE_CHK, 1668 rt->dst.flags); 1669 1670 if (new_rt) { 1671 new_rt->rt_genid = rt_genid_ipv4(dev_net(dev)); 1672 new_rt->rt_flags = rt->rt_flags; 1673 new_rt->rt_type = rt->rt_type; 1674 new_rt->rt_is_input = rt->rt_is_input; 1675 new_rt->rt_iif = rt->rt_iif; 1676 new_rt->rt_pmtu = rt->rt_pmtu; 1677 new_rt->rt_mtu_locked = rt->rt_mtu_locked; 1678 new_rt->rt_gw_family = rt->rt_gw_family; 1679 if (rt->rt_gw_family == AF_INET) 1680 new_rt->rt_gw4 = rt->rt_gw4; 1681 else if (rt->rt_gw_family == AF_INET6) 1682 new_rt->rt_gw6 = rt->rt_gw6; 1683 INIT_LIST_HEAD(&new_rt->rt_uncached); 1684 1685 new_rt->dst.input = rt->dst.input; 1686 new_rt->dst.output = rt->dst.output; 1687 new_rt->dst.error = rt->dst.error; 1688 new_rt->dst.lastuse = jiffies; 1689 new_rt->dst.lwtstate = lwtstate_get(rt->dst.lwtstate); 1690 } 1691 return new_rt; 1692 } 1693 EXPORT_SYMBOL(rt_dst_clone); 1694 1695 /* called in rcu_read_lock() section */ 1696 int ip_mc_validate_source(struct sk_buff *skb, __be32 daddr, __be32 saddr, 1697 u8 tos, struct net_device *dev, 1698 struct in_device *in_dev, u32 *itag) 1699 { 1700 int err; 1701 1702 /* Primary sanity checks. */ 1703 if (!in_dev) 1704 return -EINVAL; 1705 1706 if (ipv4_is_multicast(saddr) || ipv4_is_lbcast(saddr) || 1707 skb->protocol != htons(ETH_P_IP)) 1708 return -EINVAL; 1709 1710 if (ipv4_is_loopback(saddr) && !IN_DEV_ROUTE_LOCALNET(in_dev)) 1711 return -EINVAL; 1712 1713 if (ipv4_is_zeronet(saddr)) { 1714 if (!ipv4_is_local_multicast(daddr) && 1715 ip_hdr(skb)->protocol != IPPROTO_IGMP) 1716 return -EINVAL; 1717 } else { 1718 err = fib_validate_source(skb, saddr, 0, tos, 0, dev, 1719 in_dev, itag); 1720 if (err < 0) 1721 return err; 1722 } 1723 return 0; 1724 } 1725 1726 /* called in rcu_read_lock() section */ 1727 static int ip_route_input_mc(struct sk_buff *skb, __be32 daddr, __be32 saddr, 1728 u8 tos, struct net_device *dev, int our) 1729 { 1730 struct in_device *in_dev = __in_dev_get_rcu(dev); 1731 unsigned int flags = RTCF_MULTICAST; 1732 struct rtable *rth; 1733 u32 itag = 0; 1734 int err; 1735 1736 err = ip_mc_validate_source(skb, daddr, saddr, tos, dev, in_dev, &itag); 1737 if (err) 1738 return err; 1739 1740 if (our) 1741 flags |= RTCF_LOCAL; 1742 1743 rth = rt_dst_alloc(dev_net(dev)->loopback_dev, flags, RTN_MULTICAST, 1744 IN_DEV_ORCONF(in_dev, NOPOLICY), false); 1745 if (!rth) 1746 return -ENOBUFS; 1747 1748 #ifdef CONFIG_IP_ROUTE_CLASSID 1749 rth->dst.tclassid = itag; 1750 #endif 1751 rth->dst.output = ip_rt_bug; 1752 rth->rt_is_input= 1; 1753 1754 #ifdef CONFIG_IP_MROUTE 1755 if (!ipv4_is_local_multicast(daddr) && IN_DEV_MFORWARD(in_dev)) 1756 rth->dst.input = ip_mr_input; 1757 #endif 1758 RT_CACHE_STAT_INC(in_slow_mc); 1759 1760 skb_dst_set(skb, &rth->dst); 1761 return 0; 1762 } 1763 1764 1765 static void ip_handle_martian_source(struct net_device *dev, 1766 struct in_device *in_dev, 1767 struct sk_buff *skb, 1768 __be32 daddr, 1769 __be32 saddr) 1770 { 1771 RT_CACHE_STAT_INC(in_martian_src); 1772 #ifdef CONFIG_IP_ROUTE_VERBOSE 1773 if (IN_DEV_LOG_MARTIANS(in_dev) && net_ratelimit()) { 1774 /* 1775 * RFC1812 recommendation, if source is martian, 1776 * the only hint is MAC header. 1777 */ 1778 pr_warn("martian source %pI4 from %pI4, on dev %s\n", 1779 &daddr, &saddr, dev->name); 1780 if (dev->hard_header_len && skb_mac_header_was_set(skb)) { 1781 print_hex_dump(KERN_WARNING, "ll header: ", 1782 DUMP_PREFIX_OFFSET, 16, 1, 1783 skb_mac_header(skb), 1784 dev->hard_header_len, false); 1785 } 1786 } 1787 #endif 1788 } 1789 1790 /* called in rcu_read_lock() section */ 1791 static int __mkroute_input(struct sk_buff *skb, 1792 const struct fib_result *res, 1793 struct in_device *in_dev, 1794 __be32 daddr, __be32 saddr, u32 tos) 1795 { 1796 struct fib_nh_common *nhc = FIB_RES_NHC(*res); 1797 struct net_device *dev = nhc->nhc_dev; 1798 struct fib_nh_exception *fnhe; 1799 struct rtable *rth; 1800 int err; 1801 struct in_device *out_dev; 1802 bool do_cache; 1803 u32 itag = 0; 1804 1805 /* get a working reference to the output device */ 1806 out_dev = __in_dev_get_rcu(dev); 1807 if (!out_dev) { 1808 net_crit_ratelimited("Bug in ip_route_input_slow(). Please report.\n"); 1809 return -EINVAL; 1810 } 1811 1812 err = fib_validate_source(skb, saddr, daddr, tos, FIB_RES_OIF(*res), 1813 in_dev->dev, in_dev, &itag); 1814 if (err < 0) { 1815 ip_handle_martian_source(in_dev->dev, in_dev, skb, daddr, 1816 saddr); 1817 1818 goto cleanup; 1819 } 1820 1821 do_cache = res->fi && !itag; 1822 if (out_dev == in_dev && err && IN_DEV_TX_REDIRECTS(out_dev) && 1823 skb->protocol == htons(ETH_P_IP)) { 1824 __be32 gw; 1825 1826 gw = nhc->nhc_gw_family == AF_INET ? nhc->nhc_gw.ipv4 : 0; 1827 if (IN_DEV_SHARED_MEDIA(out_dev) || 1828 inet_addr_onlink(out_dev, saddr, gw)) 1829 IPCB(skb)->flags |= IPSKB_DOREDIRECT; 1830 } 1831 1832 if (skb->protocol != htons(ETH_P_IP)) { 1833 /* Not IP (i.e. ARP). Do not create route, if it is 1834 * invalid for proxy arp. DNAT routes are always valid. 1835 * 1836 * Proxy arp feature have been extended to allow, ARP 1837 * replies back to the same interface, to support 1838 * Private VLAN switch technologies. See arp.c. 1839 */ 1840 if (out_dev == in_dev && 1841 IN_DEV_PROXY_ARP_PVLAN(in_dev) == 0) { 1842 err = -EINVAL; 1843 goto cleanup; 1844 } 1845 } 1846 1847 fnhe = find_exception(nhc, daddr); 1848 if (do_cache) { 1849 if (fnhe) 1850 rth = rcu_dereference(fnhe->fnhe_rth_input); 1851 else 1852 rth = rcu_dereference(nhc->nhc_rth_input); 1853 if (rt_cache_valid(rth)) { 1854 skb_dst_set_noref(skb, &rth->dst); 1855 goto out; 1856 } 1857 } 1858 1859 rth = rt_dst_alloc(out_dev->dev, 0, res->type, 1860 IN_DEV_ORCONF(in_dev, NOPOLICY), 1861 IN_DEV_ORCONF(out_dev, NOXFRM)); 1862 if (!rth) { 1863 err = -ENOBUFS; 1864 goto cleanup; 1865 } 1866 1867 rth->rt_is_input = 1; 1868 RT_CACHE_STAT_INC(in_slow_tot); 1869 1870 rth->dst.input = ip_forward; 1871 1872 rt_set_nexthop(rth, daddr, res, fnhe, res->fi, res->type, itag, 1873 do_cache); 1874 lwtunnel_set_redirect(&rth->dst); 1875 skb_dst_set(skb, &rth->dst); 1876 out: 1877 err = 0; 1878 cleanup: 1879 return err; 1880 } 1881 1882 #ifdef CONFIG_IP_ROUTE_MULTIPATH 1883 /* To make ICMP packets follow the right flow, the multipath hash is 1884 * calculated from the inner IP addresses. 1885 */ 1886 static void ip_multipath_l3_keys(const struct sk_buff *skb, 1887 struct flow_keys *hash_keys) 1888 { 1889 const struct iphdr *outer_iph = ip_hdr(skb); 1890 const struct iphdr *key_iph = outer_iph; 1891 const struct iphdr *inner_iph; 1892 const struct icmphdr *icmph; 1893 struct iphdr _inner_iph; 1894 struct icmphdr _icmph; 1895 1896 if (likely(outer_iph->protocol != IPPROTO_ICMP)) 1897 goto out; 1898 1899 if (unlikely((outer_iph->frag_off & htons(IP_OFFSET)) != 0)) 1900 goto out; 1901 1902 icmph = skb_header_pointer(skb, outer_iph->ihl * 4, sizeof(_icmph), 1903 &_icmph); 1904 if (!icmph) 1905 goto out; 1906 1907 if (!icmp_is_err(icmph->type)) 1908 goto out; 1909 1910 inner_iph = skb_header_pointer(skb, 1911 outer_iph->ihl * 4 + sizeof(_icmph), 1912 sizeof(_inner_iph), &_inner_iph); 1913 if (!inner_iph) 1914 goto out; 1915 1916 key_iph = inner_iph; 1917 out: 1918 hash_keys->addrs.v4addrs.src = key_iph->saddr; 1919 hash_keys->addrs.v4addrs.dst = key_iph->daddr; 1920 } 1921 1922 /* if skb is set it will be used and fl4 can be NULL */ 1923 int fib_multipath_hash(const struct net *net, const struct flowi4 *fl4, 1924 const struct sk_buff *skb, struct flow_keys *flkeys) 1925 { 1926 u32 multipath_hash = fl4 ? fl4->flowi4_multipath_hash : 0; 1927 struct flow_keys hash_keys; 1928 u32 mhash; 1929 1930 switch (net->ipv4.sysctl_fib_multipath_hash_policy) { 1931 case 0: 1932 memset(&hash_keys, 0, sizeof(hash_keys)); 1933 hash_keys.control.addr_type = FLOW_DISSECTOR_KEY_IPV4_ADDRS; 1934 if (skb) { 1935 ip_multipath_l3_keys(skb, &hash_keys); 1936 } else { 1937 hash_keys.addrs.v4addrs.src = fl4->saddr; 1938 hash_keys.addrs.v4addrs.dst = fl4->daddr; 1939 } 1940 break; 1941 case 1: 1942 /* skb is currently provided only when forwarding */ 1943 if (skb) { 1944 unsigned int flag = FLOW_DISSECTOR_F_STOP_AT_ENCAP; 1945 struct flow_keys keys; 1946 1947 /* short-circuit if we already have L4 hash present */ 1948 if (skb->l4_hash) 1949 return skb_get_hash_raw(skb) >> 1; 1950 1951 memset(&hash_keys, 0, sizeof(hash_keys)); 1952 1953 if (!flkeys) { 1954 skb_flow_dissect_flow_keys(skb, &keys, flag); 1955 flkeys = &keys; 1956 } 1957 1958 hash_keys.control.addr_type = FLOW_DISSECTOR_KEY_IPV4_ADDRS; 1959 hash_keys.addrs.v4addrs.src = flkeys->addrs.v4addrs.src; 1960 hash_keys.addrs.v4addrs.dst = flkeys->addrs.v4addrs.dst; 1961 hash_keys.ports.src = flkeys->ports.src; 1962 hash_keys.ports.dst = flkeys->ports.dst; 1963 hash_keys.basic.ip_proto = flkeys->basic.ip_proto; 1964 } else { 1965 memset(&hash_keys, 0, sizeof(hash_keys)); 1966 hash_keys.control.addr_type = FLOW_DISSECTOR_KEY_IPV4_ADDRS; 1967 hash_keys.addrs.v4addrs.src = fl4->saddr; 1968 hash_keys.addrs.v4addrs.dst = fl4->daddr; 1969 hash_keys.ports.src = fl4->fl4_sport; 1970 hash_keys.ports.dst = fl4->fl4_dport; 1971 hash_keys.basic.ip_proto = fl4->flowi4_proto; 1972 } 1973 break; 1974 case 2: 1975 memset(&hash_keys, 0, sizeof(hash_keys)); 1976 /* skb is currently provided only when forwarding */ 1977 if (skb) { 1978 struct flow_keys keys; 1979 1980 skb_flow_dissect_flow_keys(skb, &keys, 0); 1981 /* Inner can be v4 or v6 */ 1982 if (keys.control.addr_type == FLOW_DISSECTOR_KEY_IPV4_ADDRS) { 1983 hash_keys.control.addr_type = FLOW_DISSECTOR_KEY_IPV4_ADDRS; 1984 hash_keys.addrs.v4addrs.src = keys.addrs.v4addrs.src; 1985 hash_keys.addrs.v4addrs.dst = keys.addrs.v4addrs.dst; 1986 } else if (keys.control.addr_type == FLOW_DISSECTOR_KEY_IPV6_ADDRS) { 1987 hash_keys.control.addr_type = FLOW_DISSECTOR_KEY_IPV6_ADDRS; 1988 hash_keys.addrs.v6addrs.src = keys.addrs.v6addrs.src; 1989 hash_keys.addrs.v6addrs.dst = keys.addrs.v6addrs.dst; 1990 hash_keys.tags.flow_label = keys.tags.flow_label; 1991 hash_keys.basic.ip_proto = keys.basic.ip_proto; 1992 } else { 1993 /* Same as case 0 */ 1994 hash_keys.control.addr_type = FLOW_DISSECTOR_KEY_IPV4_ADDRS; 1995 ip_multipath_l3_keys(skb, &hash_keys); 1996 } 1997 } else { 1998 /* Same as case 0 */ 1999 hash_keys.control.addr_type = FLOW_DISSECTOR_KEY_IPV4_ADDRS; 2000 hash_keys.addrs.v4addrs.src = fl4->saddr; 2001 hash_keys.addrs.v4addrs.dst = fl4->daddr; 2002 } 2003 break; 2004 } 2005 mhash = flow_hash_from_keys(&hash_keys); 2006 2007 if (multipath_hash) 2008 mhash = jhash_2words(mhash, multipath_hash, 0); 2009 2010 return mhash >> 1; 2011 } 2012 #endif /* CONFIG_IP_ROUTE_MULTIPATH */ 2013 2014 static int ip_mkroute_input(struct sk_buff *skb, 2015 struct fib_result *res, 2016 struct in_device *in_dev, 2017 __be32 daddr, __be32 saddr, u32 tos, 2018 struct flow_keys *hkeys) 2019 { 2020 #ifdef CONFIG_IP_ROUTE_MULTIPATH 2021 if (res->fi && fib_info_num_path(res->fi) > 1) { 2022 int h = fib_multipath_hash(res->fi->fib_net, NULL, skb, hkeys); 2023 2024 fib_select_multipath(res, h); 2025 } 2026 #endif 2027 2028 /* create a routing cache entry */ 2029 return __mkroute_input(skb, res, in_dev, daddr, saddr, tos); 2030 } 2031 2032 /* Implements all the saddr-related checks as ip_route_input_slow(), 2033 * assuming daddr is valid and the destination is not a local broadcast one. 2034 * Uses the provided hint instead of performing a route lookup. 2035 */ 2036 int ip_route_use_hint(struct sk_buff *skb, __be32 daddr, __be32 saddr, 2037 u8 tos, struct net_device *dev, 2038 const struct sk_buff *hint) 2039 { 2040 struct in_device *in_dev = __in_dev_get_rcu(dev); 2041 struct rtable *rt = skb_rtable(hint); 2042 struct net *net = dev_net(dev); 2043 int err = -EINVAL; 2044 u32 tag = 0; 2045 2046 if (ipv4_is_multicast(saddr) || ipv4_is_lbcast(saddr)) 2047 goto martian_source; 2048 2049 if (ipv4_is_zeronet(saddr)) 2050 goto martian_source; 2051 2052 if (ipv4_is_loopback(saddr) && !IN_DEV_NET_ROUTE_LOCALNET(in_dev, net)) 2053 goto martian_source; 2054 2055 if (rt->rt_type != RTN_LOCAL) 2056 goto skip_validate_source; 2057 2058 tos &= IPTOS_RT_MASK; 2059 err = fib_validate_source(skb, saddr, daddr, tos, 0, dev, in_dev, &tag); 2060 if (err < 0) 2061 goto martian_source; 2062 2063 skip_validate_source: 2064 skb_dst_copy(skb, hint); 2065 return 0; 2066 2067 martian_source: 2068 ip_handle_martian_source(dev, in_dev, skb, daddr, saddr); 2069 return err; 2070 } 2071 2072 /* 2073 * NOTE. We drop all the packets that has local source 2074 * addresses, because every properly looped back packet 2075 * must have correct destination already attached by output routine. 2076 * Changes in the enforced policies must be applied also to 2077 * ip_route_use_hint(). 2078 * 2079 * Such approach solves two big problems: 2080 * 1. Not simplex devices are handled properly. 2081 * 2. IP spoofing attempts are filtered with 100% of guarantee. 2082 * called with rcu_read_lock() 2083 */ 2084 2085 static int ip_route_input_slow(struct sk_buff *skb, __be32 daddr, __be32 saddr, 2086 u8 tos, struct net_device *dev, 2087 struct fib_result *res) 2088 { 2089 struct in_device *in_dev = __in_dev_get_rcu(dev); 2090 struct flow_keys *flkeys = NULL, _flkeys; 2091 struct net *net = dev_net(dev); 2092 struct ip_tunnel_info *tun_info; 2093 int err = -EINVAL; 2094 unsigned int flags = 0; 2095 u32 itag = 0; 2096 struct rtable *rth; 2097 struct flowi4 fl4; 2098 bool do_cache = true; 2099 2100 /* IP on this device is disabled. */ 2101 2102 if (!in_dev) 2103 goto out; 2104 2105 /* Check for the most weird martians, which can be not detected 2106 by fib_lookup. 2107 */ 2108 2109 tun_info = skb_tunnel_info(skb); 2110 if (tun_info && !(tun_info->mode & IP_TUNNEL_INFO_TX)) 2111 fl4.flowi4_tun_key.tun_id = tun_info->key.tun_id; 2112 else 2113 fl4.flowi4_tun_key.tun_id = 0; 2114 skb_dst_drop(skb); 2115 2116 if (ipv4_is_multicast(saddr) || ipv4_is_lbcast(saddr)) 2117 goto martian_source; 2118 2119 res->fi = NULL; 2120 res->table = NULL; 2121 if (ipv4_is_lbcast(daddr) || (saddr == 0 && daddr == 0)) 2122 goto brd_input; 2123 2124 /* Accept zero addresses only to limited broadcast; 2125 * I even do not know to fix it or not. Waiting for complains :-) 2126 */ 2127 if (ipv4_is_zeronet(saddr)) 2128 goto martian_source; 2129 2130 if (ipv4_is_zeronet(daddr)) 2131 goto martian_destination; 2132 2133 /* Following code try to avoid calling IN_DEV_NET_ROUTE_LOCALNET(), 2134 * and call it once if daddr or/and saddr are loopback addresses 2135 */ 2136 if (ipv4_is_loopback(daddr)) { 2137 if (!IN_DEV_NET_ROUTE_LOCALNET(in_dev, net)) 2138 goto martian_destination; 2139 } else if (ipv4_is_loopback(saddr)) { 2140 if (!IN_DEV_NET_ROUTE_LOCALNET(in_dev, net)) 2141 goto martian_source; 2142 } 2143 2144 /* 2145 * Now we are ready to route packet. 2146 */ 2147 fl4.flowi4_oif = 0; 2148 fl4.flowi4_iif = dev->ifindex; 2149 fl4.flowi4_mark = skb->mark; 2150 fl4.flowi4_tos = tos; 2151 fl4.flowi4_scope = RT_SCOPE_UNIVERSE; 2152 fl4.flowi4_flags = 0; 2153 fl4.daddr = daddr; 2154 fl4.saddr = saddr; 2155 fl4.flowi4_uid = sock_net_uid(net, NULL); 2156 fl4.flowi4_multipath_hash = 0; 2157 2158 if (fib4_rules_early_flow_dissect(net, skb, &fl4, &_flkeys)) { 2159 flkeys = &_flkeys; 2160 } else { 2161 fl4.flowi4_proto = 0; 2162 fl4.fl4_sport = 0; 2163 fl4.fl4_dport = 0; 2164 } 2165 2166 err = fib_lookup(net, &fl4, res, 0); 2167 if (err != 0) { 2168 if (!IN_DEV_FORWARD(in_dev)) 2169 err = -EHOSTUNREACH; 2170 goto no_route; 2171 } 2172 2173 if (res->type == RTN_BROADCAST) { 2174 if (IN_DEV_BFORWARD(in_dev)) 2175 goto make_route; 2176 /* not do cache if bc_forwarding is enabled */ 2177 if (IPV4_DEVCONF_ALL(net, BC_FORWARDING)) 2178 do_cache = false; 2179 goto brd_input; 2180 } 2181 2182 if (res->type == RTN_LOCAL) { 2183 err = fib_validate_source(skb, saddr, daddr, tos, 2184 0, dev, in_dev, &itag); 2185 if (err < 0) 2186 goto martian_source; 2187 goto local_input; 2188 } 2189 2190 if (!IN_DEV_FORWARD(in_dev)) { 2191 err = -EHOSTUNREACH; 2192 goto no_route; 2193 } 2194 if (res->type != RTN_UNICAST) 2195 goto martian_destination; 2196 2197 make_route: 2198 err = ip_mkroute_input(skb, res, in_dev, daddr, saddr, tos, flkeys); 2199 out: return err; 2200 2201 brd_input: 2202 if (skb->protocol != htons(ETH_P_IP)) 2203 goto e_inval; 2204 2205 if (!ipv4_is_zeronet(saddr)) { 2206 err = fib_validate_source(skb, saddr, 0, tos, 0, dev, 2207 in_dev, &itag); 2208 if (err < 0) 2209 goto martian_source; 2210 } 2211 flags |= RTCF_BROADCAST; 2212 res->type = RTN_BROADCAST; 2213 RT_CACHE_STAT_INC(in_brd); 2214 2215 local_input: 2216 do_cache &= res->fi && !itag; 2217 if (do_cache) { 2218 struct fib_nh_common *nhc = FIB_RES_NHC(*res); 2219 2220 rth = rcu_dereference(nhc->nhc_rth_input); 2221 if (rt_cache_valid(rth)) { 2222 skb_dst_set_noref(skb, &rth->dst); 2223 err = 0; 2224 goto out; 2225 } 2226 } 2227 2228 rth = rt_dst_alloc(l3mdev_master_dev_rcu(dev) ? : net->loopback_dev, 2229 flags | RTCF_LOCAL, res->type, 2230 IN_DEV_ORCONF(in_dev, NOPOLICY), false); 2231 if (!rth) 2232 goto e_nobufs; 2233 2234 rth->dst.output= ip_rt_bug; 2235 #ifdef CONFIG_IP_ROUTE_CLASSID 2236 rth->dst.tclassid = itag; 2237 #endif 2238 rth->rt_is_input = 1; 2239 2240 RT_CACHE_STAT_INC(in_slow_tot); 2241 if (res->type == RTN_UNREACHABLE) { 2242 rth->dst.input= ip_error; 2243 rth->dst.error= -err; 2244 rth->rt_flags &= ~RTCF_LOCAL; 2245 } 2246 2247 if (do_cache) { 2248 struct fib_nh_common *nhc = FIB_RES_NHC(*res); 2249 2250 rth->dst.lwtstate = lwtstate_get(nhc->nhc_lwtstate); 2251 if (lwtunnel_input_redirect(rth->dst.lwtstate)) { 2252 WARN_ON(rth->dst.input == lwtunnel_input); 2253 rth->dst.lwtstate->orig_input = rth->dst.input; 2254 rth->dst.input = lwtunnel_input; 2255 } 2256 2257 if (unlikely(!rt_cache_route(nhc, rth))) 2258 rt_add_uncached_list(rth); 2259 } 2260 skb_dst_set(skb, &rth->dst); 2261 err = 0; 2262 goto out; 2263 2264 no_route: 2265 RT_CACHE_STAT_INC(in_no_route); 2266 res->type = RTN_UNREACHABLE; 2267 res->fi = NULL; 2268 res->table = NULL; 2269 goto local_input; 2270 2271 /* 2272 * Do not cache martian addresses: they should be logged (RFC1812) 2273 */ 2274 martian_destination: 2275 RT_CACHE_STAT_INC(in_martian_dst); 2276 #ifdef CONFIG_IP_ROUTE_VERBOSE 2277 if (IN_DEV_LOG_MARTIANS(in_dev)) 2278 net_warn_ratelimited("martian destination %pI4 from %pI4, dev %s\n", 2279 &daddr, &saddr, dev->name); 2280 #endif 2281 2282 e_inval: 2283 err = -EINVAL; 2284 goto out; 2285 2286 e_nobufs: 2287 err = -ENOBUFS; 2288 goto out; 2289 2290 martian_source: 2291 ip_handle_martian_source(dev, in_dev, skb, daddr, saddr); 2292 goto out; 2293 } 2294 2295 int ip_route_input_noref(struct sk_buff *skb, __be32 daddr, __be32 saddr, 2296 u8 tos, struct net_device *dev) 2297 { 2298 struct fib_result res; 2299 int err; 2300 2301 tos &= IPTOS_RT_MASK; 2302 rcu_read_lock(); 2303 err = ip_route_input_rcu(skb, daddr, saddr, tos, dev, &res); 2304 rcu_read_unlock(); 2305 2306 return err; 2307 } 2308 EXPORT_SYMBOL(ip_route_input_noref); 2309 2310 /* called with rcu_read_lock held */ 2311 int ip_route_input_rcu(struct sk_buff *skb, __be32 daddr, __be32 saddr, 2312 u8 tos, struct net_device *dev, struct fib_result *res) 2313 { 2314 /* Multicast recognition logic is moved from route cache to here. 2315 The problem was that too many Ethernet cards have broken/missing 2316 hardware multicast filters :-( As result the host on multicasting 2317 network acquires a lot of useless route cache entries, sort of 2318 SDR messages from all the world. Now we try to get rid of them. 2319 Really, provided software IP multicast filter is organized 2320 reasonably (at least, hashed), it does not result in a slowdown 2321 comparing with route cache reject entries. 2322 Note, that multicast routers are not affected, because 2323 route cache entry is created eventually. 2324 */ 2325 if (ipv4_is_multicast(daddr)) { 2326 struct in_device *in_dev = __in_dev_get_rcu(dev); 2327 int our = 0; 2328 int err = -EINVAL; 2329 2330 if (!in_dev) 2331 return err; 2332 our = ip_check_mc_rcu(in_dev, daddr, saddr, 2333 ip_hdr(skb)->protocol); 2334 2335 /* check l3 master if no match yet */ 2336 if (!our && netif_is_l3_slave(dev)) { 2337 struct in_device *l3_in_dev; 2338 2339 l3_in_dev = __in_dev_get_rcu(skb->dev); 2340 if (l3_in_dev) 2341 our = ip_check_mc_rcu(l3_in_dev, daddr, saddr, 2342 ip_hdr(skb)->protocol); 2343 } 2344 2345 if (our 2346 #ifdef CONFIG_IP_MROUTE 2347 || 2348 (!ipv4_is_local_multicast(daddr) && 2349 IN_DEV_MFORWARD(in_dev)) 2350 #endif 2351 ) { 2352 err = ip_route_input_mc(skb, daddr, saddr, 2353 tos, dev, our); 2354 } 2355 return err; 2356 } 2357 2358 return ip_route_input_slow(skb, daddr, saddr, tos, dev, res); 2359 } 2360 2361 /* called with rcu_read_lock() */ 2362 static struct rtable *__mkroute_output(const struct fib_result *res, 2363 const struct flowi4 *fl4, int orig_oif, 2364 struct net_device *dev_out, 2365 unsigned int flags) 2366 { 2367 struct fib_info *fi = res->fi; 2368 struct fib_nh_exception *fnhe; 2369 struct in_device *in_dev; 2370 u16 type = res->type; 2371 struct rtable *rth; 2372 bool do_cache; 2373 2374 in_dev = __in_dev_get_rcu(dev_out); 2375 if (!in_dev) 2376 return ERR_PTR(-EINVAL); 2377 2378 if (likely(!IN_DEV_ROUTE_LOCALNET(in_dev))) 2379 if (ipv4_is_loopback(fl4->saddr) && 2380 !(dev_out->flags & IFF_LOOPBACK) && 2381 !netif_is_l3_master(dev_out)) 2382 return ERR_PTR(-EINVAL); 2383 2384 if (ipv4_is_lbcast(fl4->daddr)) 2385 type = RTN_BROADCAST; 2386 else if (ipv4_is_multicast(fl4->daddr)) 2387 type = RTN_MULTICAST; 2388 else if (ipv4_is_zeronet(fl4->daddr)) 2389 return ERR_PTR(-EINVAL); 2390 2391 if (dev_out->flags & IFF_LOOPBACK) 2392 flags |= RTCF_LOCAL; 2393 2394 do_cache = true; 2395 if (type == RTN_BROADCAST) { 2396 flags |= RTCF_BROADCAST | RTCF_LOCAL; 2397 fi = NULL; 2398 } else if (type == RTN_MULTICAST) { 2399 flags |= RTCF_MULTICAST | RTCF_LOCAL; 2400 if (!ip_check_mc_rcu(in_dev, fl4->daddr, fl4->saddr, 2401 fl4->flowi4_proto)) 2402 flags &= ~RTCF_LOCAL; 2403 else 2404 do_cache = false; 2405 /* If multicast route do not exist use 2406 * default one, but do not gateway in this case. 2407 * Yes, it is hack. 2408 */ 2409 if (fi && res->prefixlen < 4) 2410 fi = NULL; 2411 } else if ((type == RTN_LOCAL) && (orig_oif != 0) && 2412 (orig_oif != dev_out->ifindex)) { 2413 /* For local routes that require a particular output interface 2414 * we do not want to cache the result. Caching the result 2415 * causes incorrect behaviour when there are multiple source 2416 * addresses on the interface, the end result being that if the 2417 * intended recipient is waiting on that interface for the 2418 * packet he won't receive it because it will be delivered on 2419 * the loopback interface and the IP_PKTINFO ipi_ifindex will 2420 * be set to the loopback interface as well. 2421 */ 2422 do_cache = false; 2423 } 2424 2425 fnhe = NULL; 2426 do_cache &= fi != NULL; 2427 if (fi) { 2428 struct fib_nh_common *nhc = FIB_RES_NHC(*res); 2429 struct rtable __rcu **prth; 2430 2431 fnhe = find_exception(nhc, fl4->daddr); 2432 if (!do_cache) 2433 goto add; 2434 if (fnhe) { 2435 prth = &fnhe->fnhe_rth_output; 2436 } else { 2437 if (unlikely(fl4->flowi4_flags & 2438 FLOWI_FLAG_KNOWN_NH && 2439 !(nhc->nhc_gw_family && 2440 nhc->nhc_scope == RT_SCOPE_LINK))) { 2441 do_cache = false; 2442 goto add; 2443 } 2444 prth = raw_cpu_ptr(nhc->nhc_pcpu_rth_output); 2445 } 2446 rth = rcu_dereference(*prth); 2447 if (rt_cache_valid(rth) && dst_hold_safe(&rth->dst)) 2448 return rth; 2449 } 2450 2451 add: 2452 rth = rt_dst_alloc(dev_out, flags, type, 2453 IN_DEV_ORCONF(in_dev, NOPOLICY), 2454 IN_DEV_ORCONF(in_dev, NOXFRM)); 2455 if (!rth) 2456 return ERR_PTR(-ENOBUFS); 2457 2458 rth->rt_iif = orig_oif; 2459 2460 RT_CACHE_STAT_INC(out_slow_tot); 2461 2462 if (flags & (RTCF_BROADCAST | RTCF_MULTICAST)) { 2463 if (flags & RTCF_LOCAL && 2464 !(dev_out->flags & IFF_LOOPBACK)) { 2465 rth->dst.output = ip_mc_output; 2466 RT_CACHE_STAT_INC(out_slow_mc); 2467 } 2468 #ifdef CONFIG_IP_MROUTE 2469 if (type == RTN_MULTICAST) { 2470 if (IN_DEV_MFORWARD(in_dev) && 2471 !ipv4_is_local_multicast(fl4->daddr)) { 2472 rth->dst.input = ip_mr_input; 2473 rth->dst.output = ip_mc_output; 2474 } 2475 } 2476 #endif 2477 } 2478 2479 rt_set_nexthop(rth, fl4->daddr, res, fnhe, fi, type, 0, do_cache); 2480 lwtunnel_set_redirect(&rth->dst); 2481 2482 return rth; 2483 } 2484 2485 /* 2486 * Major route resolver routine. 2487 */ 2488 2489 struct rtable *ip_route_output_key_hash(struct net *net, struct flowi4 *fl4, 2490 const struct sk_buff *skb) 2491 { 2492 __u8 tos = RT_FL_TOS(fl4); 2493 struct fib_result res = { 2494 .type = RTN_UNSPEC, 2495 .fi = NULL, 2496 .table = NULL, 2497 .tclassid = 0, 2498 }; 2499 struct rtable *rth; 2500 2501 fl4->flowi4_iif = LOOPBACK_IFINDEX; 2502 fl4->flowi4_tos = tos & IPTOS_RT_MASK; 2503 fl4->flowi4_scope = ((tos & RTO_ONLINK) ? 2504 RT_SCOPE_LINK : RT_SCOPE_UNIVERSE); 2505 2506 rcu_read_lock(); 2507 rth = ip_route_output_key_hash_rcu(net, fl4, &res, skb); 2508 rcu_read_unlock(); 2509 2510 return rth; 2511 } 2512 EXPORT_SYMBOL_GPL(ip_route_output_key_hash); 2513 2514 struct rtable *ip_route_output_key_hash_rcu(struct net *net, struct flowi4 *fl4, 2515 struct fib_result *res, 2516 const struct sk_buff *skb) 2517 { 2518 struct net_device *dev_out = NULL; 2519 int orig_oif = fl4->flowi4_oif; 2520 unsigned int flags = 0; 2521 struct rtable *rth; 2522 int err; 2523 2524 if (fl4->saddr) { 2525 if (ipv4_is_multicast(fl4->saddr) || 2526 ipv4_is_lbcast(fl4->saddr) || 2527 ipv4_is_zeronet(fl4->saddr)) { 2528 rth = ERR_PTR(-EINVAL); 2529 goto out; 2530 } 2531 2532 rth = ERR_PTR(-ENETUNREACH); 2533 2534 /* I removed check for oif == dev_out->oif here. 2535 It was wrong for two reasons: 2536 1. ip_dev_find(net, saddr) can return wrong iface, if saddr 2537 is assigned to multiple interfaces. 2538 2. Moreover, we are allowed to send packets with saddr 2539 of another iface. --ANK 2540 */ 2541 2542 if (fl4->flowi4_oif == 0 && 2543 (ipv4_is_multicast(fl4->daddr) || 2544 ipv4_is_lbcast(fl4->daddr))) { 2545 /* It is equivalent to inet_addr_type(saddr) == RTN_LOCAL */ 2546 dev_out = __ip_dev_find(net, fl4->saddr, false); 2547 if (!dev_out) 2548 goto out; 2549 2550 /* Special hack: user can direct multicasts 2551 and limited broadcast via necessary interface 2552 without fiddling with IP_MULTICAST_IF or IP_PKTINFO. 2553 This hack is not just for fun, it allows 2554 vic,vat and friends to work. 2555 They bind socket to loopback, set ttl to zero 2556 and expect that it will work. 2557 From the viewpoint of routing cache they are broken, 2558 because we are not allowed to build multicast path 2559 with loopback source addr (look, routing cache 2560 cannot know, that ttl is zero, so that packet 2561 will not leave this host and route is valid). 2562 Luckily, this hack is good workaround. 2563 */ 2564 2565 fl4->flowi4_oif = dev_out->ifindex; 2566 goto make_route; 2567 } 2568 2569 if (!(fl4->flowi4_flags & FLOWI_FLAG_ANYSRC)) { 2570 /* It is equivalent to inet_addr_type(saddr) == RTN_LOCAL */ 2571 if (!__ip_dev_find(net, fl4->saddr, false)) 2572 goto out; 2573 } 2574 } 2575 2576 2577 if (fl4->flowi4_oif) { 2578 dev_out = dev_get_by_index_rcu(net, fl4->flowi4_oif); 2579 rth = ERR_PTR(-ENODEV); 2580 if (!dev_out) 2581 goto out; 2582 2583 /* RACE: Check return value of inet_select_addr instead. */ 2584 if (!(dev_out->flags & IFF_UP) || !__in_dev_get_rcu(dev_out)) { 2585 rth = ERR_PTR(-ENETUNREACH); 2586 goto out; 2587 } 2588 if (ipv4_is_local_multicast(fl4->daddr) || 2589 ipv4_is_lbcast(fl4->daddr) || 2590 fl4->flowi4_proto == IPPROTO_IGMP) { 2591 if (!fl4->saddr) 2592 fl4->saddr = inet_select_addr(dev_out, 0, 2593 RT_SCOPE_LINK); 2594 goto make_route; 2595 } 2596 if (!fl4->saddr) { 2597 if (ipv4_is_multicast(fl4->daddr)) 2598 fl4->saddr = inet_select_addr(dev_out, 0, 2599 fl4->flowi4_scope); 2600 else if (!fl4->daddr) 2601 fl4->saddr = inet_select_addr(dev_out, 0, 2602 RT_SCOPE_HOST); 2603 } 2604 } 2605 2606 if (!fl4->daddr) { 2607 fl4->daddr = fl4->saddr; 2608 if (!fl4->daddr) 2609 fl4->daddr = fl4->saddr = htonl(INADDR_LOOPBACK); 2610 dev_out = net->loopback_dev; 2611 fl4->flowi4_oif = LOOPBACK_IFINDEX; 2612 res->type = RTN_LOCAL; 2613 flags |= RTCF_LOCAL; 2614 goto make_route; 2615 } 2616 2617 err = fib_lookup(net, fl4, res, 0); 2618 if (err) { 2619 res->fi = NULL; 2620 res->table = NULL; 2621 if (fl4->flowi4_oif && 2622 (ipv4_is_multicast(fl4->daddr) || 2623 !netif_index_is_l3_master(net, fl4->flowi4_oif))) { 2624 /* Apparently, routing tables are wrong. Assume, 2625 that the destination is on link. 2626 2627 WHY? DW. 2628 Because we are allowed to send to iface 2629 even if it has NO routes and NO assigned 2630 addresses. When oif is specified, routing 2631 tables are looked up with only one purpose: 2632 to catch if destination is gatewayed, rather than 2633 direct. Moreover, if MSG_DONTROUTE is set, 2634 we send packet, ignoring both routing tables 2635 and ifaddr state. --ANK 2636 2637 2638 We could make it even if oif is unknown, 2639 likely IPv6, but we do not. 2640 */ 2641 2642 if (fl4->saddr == 0) 2643 fl4->saddr = inet_select_addr(dev_out, 0, 2644 RT_SCOPE_LINK); 2645 res->type = RTN_UNICAST; 2646 goto make_route; 2647 } 2648 rth = ERR_PTR(err); 2649 goto out; 2650 } 2651 2652 if (res->type == RTN_LOCAL) { 2653 if (!fl4->saddr) { 2654 if (res->fi->fib_prefsrc) 2655 fl4->saddr = res->fi->fib_prefsrc; 2656 else 2657 fl4->saddr = fl4->daddr; 2658 } 2659 2660 /* L3 master device is the loopback for that domain */ 2661 dev_out = l3mdev_master_dev_rcu(FIB_RES_DEV(*res)) ? : 2662 net->loopback_dev; 2663 2664 /* make sure orig_oif points to fib result device even 2665 * though packet rx/tx happens over loopback or l3mdev 2666 */ 2667 orig_oif = FIB_RES_OIF(*res); 2668 2669 fl4->flowi4_oif = dev_out->ifindex; 2670 flags |= RTCF_LOCAL; 2671 goto make_route; 2672 } 2673 2674 fib_select_path(net, res, fl4, skb); 2675 2676 dev_out = FIB_RES_DEV(*res); 2677 2678 make_route: 2679 rth = __mkroute_output(res, fl4, orig_oif, dev_out, flags); 2680 2681 out: 2682 return rth; 2683 } 2684 2685 static struct dst_entry *ipv4_blackhole_dst_check(struct dst_entry *dst, u32 cookie) 2686 { 2687 return NULL; 2688 } 2689 2690 static unsigned int ipv4_blackhole_mtu(const struct dst_entry *dst) 2691 { 2692 unsigned int mtu = dst_metric_raw(dst, RTAX_MTU); 2693 2694 return mtu ? : dst->dev->mtu; 2695 } 2696 2697 static void ipv4_rt_blackhole_update_pmtu(struct dst_entry *dst, struct sock *sk, 2698 struct sk_buff *skb, u32 mtu, 2699 bool confirm_neigh) 2700 { 2701 } 2702 2703 static void ipv4_rt_blackhole_redirect(struct dst_entry *dst, struct sock *sk, 2704 struct sk_buff *skb) 2705 { 2706 } 2707 2708 static u32 *ipv4_rt_blackhole_cow_metrics(struct dst_entry *dst, 2709 unsigned long old) 2710 { 2711 return NULL; 2712 } 2713 2714 static struct dst_ops ipv4_dst_blackhole_ops = { 2715 .family = AF_INET, 2716 .check = ipv4_blackhole_dst_check, 2717 .mtu = ipv4_blackhole_mtu, 2718 .default_advmss = ipv4_default_advmss, 2719 .update_pmtu = ipv4_rt_blackhole_update_pmtu, 2720 .redirect = ipv4_rt_blackhole_redirect, 2721 .cow_metrics = ipv4_rt_blackhole_cow_metrics, 2722 .neigh_lookup = ipv4_neigh_lookup, 2723 }; 2724 2725 struct dst_entry *ipv4_blackhole_route(struct net *net, struct dst_entry *dst_orig) 2726 { 2727 struct rtable *ort = (struct rtable *) dst_orig; 2728 struct rtable *rt; 2729 2730 rt = dst_alloc(&ipv4_dst_blackhole_ops, NULL, 1, DST_OBSOLETE_DEAD, 0); 2731 if (rt) { 2732 struct dst_entry *new = &rt->dst; 2733 2734 new->__use = 1; 2735 new->input = dst_discard; 2736 new->output = dst_discard_out; 2737 2738 new->dev = net->loopback_dev; 2739 if (new->dev) 2740 dev_hold(new->dev); 2741 2742 rt->rt_is_input = ort->rt_is_input; 2743 rt->rt_iif = ort->rt_iif; 2744 rt->rt_pmtu = ort->rt_pmtu; 2745 rt->rt_mtu_locked = ort->rt_mtu_locked; 2746 2747 rt->rt_genid = rt_genid_ipv4(net); 2748 rt->rt_flags = ort->rt_flags; 2749 rt->rt_type = ort->rt_type; 2750 rt->rt_uses_gateway = ort->rt_uses_gateway; 2751 rt->rt_gw_family = ort->rt_gw_family; 2752 if (rt->rt_gw_family == AF_INET) 2753 rt->rt_gw4 = ort->rt_gw4; 2754 else if (rt->rt_gw_family == AF_INET6) 2755 rt->rt_gw6 = ort->rt_gw6; 2756 2757 INIT_LIST_HEAD(&rt->rt_uncached); 2758 } 2759 2760 dst_release(dst_orig); 2761 2762 return rt ? &rt->dst : ERR_PTR(-ENOMEM); 2763 } 2764 2765 struct rtable *ip_route_output_flow(struct net *net, struct flowi4 *flp4, 2766 const struct sock *sk) 2767 { 2768 struct rtable *rt = __ip_route_output_key(net, flp4); 2769 2770 if (IS_ERR(rt)) 2771 return rt; 2772 2773 if (flp4->flowi4_proto) { 2774 flp4->flowi4_oif = rt->dst.dev->ifindex; 2775 rt = (struct rtable *)xfrm_lookup_route(net, &rt->dst, 2776 flowi4_to_flowi(flp4), 2777 sk, 0); 2778 } 2779 2780 return rt; 2781 } 2782 EXPORT_SYMBOL_GPL(ip_route_output_flow); 2783 2784 struct rtable *ip_route_output_tunnel(struct sk_buff *skb, 2785 struct net_device *dev, 2786 struct net *net, __be32 *saddr, 2787 const struct ip_tunnel_info *info, 2788 u8 protocol, bool use_cache) 2789 { 2790 #ifdef CONFIG_DST_CACHE 2791 struct dst_cache *dst_cache; 2792 #endif 2793 struct rtable *rt = NULL; 2794 struct flowi4 fl4; 2795 __u8 tos; 2796 2797 #ifdef CONFIG_DST_CACHE 2798 dst_cache = (struct dst_cache *)&info->dst_cache; 2799 if (use_cache) { 2800 rt = dst_cache_get_ip4(dst_cache, saddr); 2801 if (rt) 2802 return rt; 2803 } 2804 #endif 2805 memset(&fl4, 0, sizeof(fl4)); 2806 fl4.flowi4_mark = skb->mark; 2807 fl4.flowi4_proto = protocol; 2808 fl4.daddr = info->key.u.ipv4.dst; 2809 fl4.saddr = info->key.u.ipv4.src; 2810 tos = info->key.tos; 2811 fl4.flowi4_tos = RT_TOS(tos); 2812 2813 rt = ip_route_output_key(net, &fl4); 2814 if (IS_ERR(rt)) { 2815 netdev_dbg(dev, "no route to %pI4\n", &fl4.daddr); 2816 return ERR_PTR(-ENETUNREACH); 2817 } 2818 if (rt->dst.dev == dev) { /* is this necessary? */ 2819 netdev_dbg(dev, "circular route to %pI4\n", &fl4.daddr); 2820 ip_rt_put(rt); 2821 return ERR_PTR(-ELOOP); 2822 } 2823 #ifdef CONFIG_DST_CACHE 2824 if (use_cache) 2825 dst_cache_set_ip4(dst_cache, &rt->dst, fl4.saddr); 2826 #endif 2827 *saddr = fl4.saddr; 2828 return rt; 2829 } 2830 EXPORT_SYMBOL_GPL(ip_route_output_tunnel); 2831 2832 /* called with rcu_read_lock held */ 2833 static int rt_fill_info(struct net *net, __be32 dst, __be32 src, 2834 struct rtable *rt, u32 table_id, struct flowi4 *fl4, 2835 struct sk_buff *skb, u32 portid, u32 seq, 2836 unsigned int flags) 2837 { 2838 struct rtmsg *r; 2839 struct nlmsghdr *nlh; 2840 unsigned long expires = 0; 2841 u32 error; 2842 u32 metrics[RTAX_MAX]; 2843 2844 nlh = nlmsg_put(skb, portid, seq, RTM_NEWROUTE, sizeof(*r), flags); 2845 if (!nlh) 2846 return -EMSGSIZE; 2847 2848 r = nlmsg_data(nlh); 2849 r->rtm_family = AF_INET; 2850 r->rtm_dst_len = 32; 2851 r->rtm_src_len = 0; 2852 r->rtm_tos = fl4 ? fl4->flowi4_tos : 0; 2853 r->rtm_table = table_id < 256 ? table_id : RT_TABLE_COMPAT; 2854 if (nla_put_u32(skb, RTA_TABLE, table_id)) 2855 goto nla_put_failure; 2856 r->rtm_type = rt->rt_type; 2857 r->rtm_scope = RT_SCOPE_UNIVERSE; 2858 r->rtm_protocol = RTPROT_UNSPEC; 2859 r->rtm_flags = (rt->rt_flags & ~0xFFFF) | RTM_F_CLONED; 2860 if (rt->rt_flags & RTCF_NOTIFY) 2861 r->rtm_flags |= RTM_F_NOTIFY; 2862 if (IPCB(skb)->flags & IPSKB_DOREDIRECT) 2863 r->rtm_flags |= RTCF_DOREDIRECT; 2864 2865 if (nla_put_in_addr(skb, RTA_DST, dst)) 2866 goto nla_put_failure; 2867 if (src) { 2868 r->rtm_src_len = 32; 2869 if (nla_put_in_addr(skb, RTA_SRC, src)) 2870 goto nla_put_failure; 2871 } 2872 if (rt->dst.dev && 2873 nla_put_u32(skb, RTA_OIF, rt->dst.dev->ifindex)) 2874 goto nla_put_failure; 2875 if (rt->dst.lwtstate && 2876 lwtunnel_fill_encap(skb, rt->dst.lwtstate, RTA_ENCAP, RTA_ENCAP_TYPE) < 0) 2877 goto nla_put_failure; 2878 #ifdef CONFIG_IP_ROUTE_CLASSID 2879 if (rt->dst.tclassid && 2880 nla_put_u32(skb, RTA_FLOW, rt->dst.tclassid)) 2881 goto nla_put_failure; 2882 #endif 2883 if (fl4 && !rt_is_input_route(rt) && 2884 fl4->saddr != src) { 2885 if (nla_put_in_addr(skb, RTA_PREFSRC, fl4->saddr)) 2886 goto nla_put_failure; 2887 } 2888 if (rt->rt_uses_gateway) { 2889 if (rt->rt_gw_family == AF_INET && 2890 nla_put_in_addr(skb, RTA_GATEWAY, rt->rt_gw4)) { 2891 goto nla_put_failure; 2892 } else if (rt->rt_gw_family == AF_INET6) { 2893 int alen = sizeof(struct in6_addr); 2894 struct nlattr *nla; 2895 struct rtvia *via; 2896 2897 nla = nla_reserve(skb, RTA_VIA, alen + 2); 2898 if (!nla) 2899 goto nla_put_failure; 2900 2901 via = nla_data(nla); 2902 via->rtvia_family = AF_INET6; 2903 memcpy(via->rtvia_addr, &rt->rt_gw6, alen); 2904 } 2905 } 2906 2907 expires = rt->dst.expires; 2908 if (expires) { 2909 unsigned long now = jiffies; 2910 2911 if (time_before(now, expires)) 2912 expires -= now; 2913 else 2914 expires = 0; 2915 } 2916 2917 memcpy(metrics, dst_metrics_ptr(&rt->dst), sizeof(metrics)); 2918 if (rt->rt_pmtu && expires) 2919 metrics[RTAX_MTU - 1] = rt->rt_pmtu; 2920 if (rt->rt_mtu_locked && expires) 2921 metrics[RTAX_LOCK - 1] |= BIT(RTAX_MTU); 2922 if (rtnetlink_put_metrics(skb, metrics) < 0) 2923 goto nla_put_failure; 2924 2925 if (fl4) { 2926 if (fl4->flowi4_mark && 2927 nla_put_u32(skb, RTA_MARK, fl4->flowi4_mark)) 2928 goto nla_put_failure; 2929 2930 if (!uid_eq(fl4->flowi4_uid, INVALID_UID) && 2931 nla_put_u32(skb, RTA_UID, 2932 from_kuid_munged(current_user_ns(), 2933 fl4->flowi4_uid))) 2934 goto nla_put_failure; 2935 2936 if (rt_is_input_route(rt)) { 2937 #ifdef CONFIG_IP_MROUTE 2938 if (ipv4_is_multicast(dst) && 2939 !ipv4_is_local_multicast(dst) && 2940 IPV4_DEVCONF_ALL(net, MC_FORWARDING)) { 2941 int err = ipmr_get_route(net, skb, 2942 fl4->saddr, fl4->daddr, 2943 r, portid); 2944 2945 if (err <= 0) { 2946 if (err == 0) 2947 return 0; 2948 goto nla_put_failure; 2949 } 2950 } else 2951 #endif 2952 if (nla_put_u32(skb, RTA_IIF, fl4->flowi4_iif)) 2953 goto nla_put_failure; 2954 } 2955 } 2956 2957 error = rt->dst.error; 2958 2959 if (rtnl_put_cacheinfo(skb, &rt->dst, 0, expires, error) < 0) 2960 goto nla_put_failure; 2961 2962 nlmsg_end(skb, nlh); 2963 return 0; 2964 2965 nla_put_failure: 2966 nlmsg_cancel(skb, nlh); 2967 return -EMSGSIZE; 2968 } 2969 2970 static int fnhe_dump_bucket(struct net *net, struct sk_buff *skb, 2971 struct netlink_callback *cb, u32 table_id, 2972 struct fnhe_hash_bucket *bucket, int genid, 2973 int *fa_index, int fa_start, unsigned int flags) 2974 { 2975 int i; 2976 2977 for (i = 0; i < FNHE_HASH_SIZE; i++) { 2978 struct fib_nh_exception *fnhe; 2979 2980 for (fnhe = rcu_dereference(bucket[i].chain); fnhe; 2981 fnhe = rcu_dereference(fnhe->fnhe_next)) { 2982 struct rtable *rt; 2983 int err; 2984 2985 if (*fa_index < fa_start) 2986 goto next; 2987 2988 if (fnhe->fnhe_genid != genid) 2989 goto next; 2990 2991 if (fnhe->fnhe_expires && 2992 time_after(jiffies, fnhe->fnhe_expires)) 2993 goto next; 2994 2995 rt = rcu_dereference(fnhe->fnhe_rth_input); 2996 if (!rt) 2997 rt = rcu_dereference(fnhe->fnhe_rth_output); 2998 if (!rt) 2999 goto next; 3000 3001 err = rt_fill_info(net, fnhe->fnhe_daddr, 0, rt, 3002 table_id, NULL, skb, 3003 NETLINK_CB(cb->skb).portid, 3004 cb->nlh->nlmsg_seq, flags); 3005 if (err) 3006 return err; 3007 next: 3008 (*fa_index)++; 3009 } 3010 } 3011 3012 return 0; 3013 } 3014 3015 int fib_dump_info_fnhe(struct sk_buff *skb, struct netlink_callback *cb, 3016 u32 table_id, struct fib_info *fi, 3017 int *fa_index, int fa_start, unsigned int flags) 3018 { 3019 struct net *net = sock_net(cb->skb->sk); 3020 int nhsel, genid = fnhe_genid(net); 3021 3022 for (nhsel = 0; nhsel < fib_info_num_path(fi); nhsel++) { 3023 struct fib_nh_common *nhc = fib_info_nhc(fi, nhsel); 3024 struct fnhe_hash_bucket *bucket; 3025 int err; 3026 3027 if (nhc->nhc_flags & RTNH_F_DEAD) 3028 continue; 3029 3030 rcu_read_lock(); 3031 bucket = rcu_dereference(nhc->nhc_exceptions); 3032 err = 0; 3033 if (bucket) 3034 err = fnhe_dump_bucket(net, skb, cb, table_id, bucket, 3035 genid, fa_index, fa_start, 3036 flags); 3037 rcu_read_unlock(); 3038 if (err) 3039 return err; 3040 } 3041 3042 return 0; 3043 } 3044 3045 static struct sk_buff *inet_rtm_getroute_build_skb(__be32 src, __be32 dst, 3046 u8 ip_proto, __be16 sport, 3047 __be16 dport) 3048 { 3049 struct sk_buff *skb; 3050 struct iphdr *iph; 3051 3052 skb = alloc_skb(NLMSG_GOODSIZE, GFP_KERNEL); 3053 if (!skb) 3054 return NULL; 3055 3056 /* Reserve room for dummy headers, this skb can pass 3057 * through good chunk of routing engine. 3058 */ 3059 skb_reset_mac_header(skb); 3060 skb_reset_network_header(skb); 3061 skb->protocol = htons(ETH_P_IP); 3062 iph = skb_put(skb, sizeof(struct iphdr)); 3063 iph->protocol = ip_proto; 3064 iph->saddr = src; 3065 iph->daddr = dst; 3066 iph->version = 0x4; 3067 iph->frag_off = 0; 3068 iph->ihl = 0x5; 3069 skb_set_transport_header(skb, skb->len); 3070 3071 switch (iph->protocol) { 3072 case IPPROTO_UDP: { 3073 struct udphdr *udph; 3074 3075 udph = skb_put_zero(skb, sizeof(struct udphdr)); 3076 udph->source = sport; 3077 udph->dest = dport; 3078 udph->len = sizeof(struct udphdr); 3079 udph->check = 0; 3080 break; 3081 } 3082 case IPPROTO_TCP: { 3083 struct tcphdr *tcph; 3084 3085 tcph = skb_put_zero(skb, sizeof(struct tcphdr)); 3086 tcph->source = sport; 3087 tcph->dest = dport; 3088 tcph->doff = sizeof(struct tcphdr) / 4; 3089 tcph->rst = 1; 3090 tcph->check = ~tcp_v4_check(sizeof(struct tcphdr), 3091 src, dst, 0); 3092 break; 3093 } 3094 case IPPROTO_ICMP: { 3095 struct icmphdr *icmph; 3096 3097 icmph = skb_put_zero(skb, sizeof(struct icmphdr)); 3098 icmph->type = ICMP_ECHO; 3099 icmph->code = 0; 3100 } 3101 } 3102 3103 return skb; 3104 } 3105 3106 static int inet_rtm_valid_getroute_req(struct sk_buff *skb, 3107 const struct nlmsghdr *nlh, 3108 struct nlattr **tb, 3109 struct netlink_ext_ack *extack) 3110 { 3111 struct rtmsg *rtm; 3112 int i, err; 3113 3114 if (nlh->nlmsg_len < nlmsg_msg_size(sizeof(*rtm))) { 3115 NL_SET_ERR_MSG(extack, 3116 "ipv4: Invalid header for route get request"); 3117 return -EINVAL; 3118 } 3119 3120 if (!netlink_strict_get_check(skb)) 3121 return nlmsg_parse_deprecated(nlh, sizeof(*rtm), tb, RTA_MAX, 3122 rtm_ipv4_policy, extack); 3123 3124 rtm = nlmsg_data(nlh); 3125 if ((rtm->rtm_src_len && rtm->rtm_src_len != 32) || 3126 (rtm->rtm_dst_len && rtm->rtm_dst_len != 32) || 3127 rtm->rtm_table || rtm->rtm_protocol || 3128 rtm->rtm_scope || rtm->rtm_type) { 3129 NL_SET_ERR_MSG(extack, "ipv4: Invalid values in header for route get request"); 3130 return -EINVAL; 3131 } 3132 3133 if (rtm->rtm_flags & ~(RTM_F_NOTIFY | 3134 RTM_F_LOOKUP_TABLE | 3135 RTM_F_FIB_MATCH)) { 3136 NL_SET_ERR_MSG(extack, "ipv4: Unsupported rtm_flags for route get request"); 3137 return -EINVAL; 3138 } 3139 3140 err = nlmsg_parse_deprecated_strict(nlh, sizeof(*rtm), tb, RTA_MAX, 3141 rtm_ipv4_policy, extack); 3142 if (err) 3143 return err; 3144 3145 if ((tb[RTA_SRC] && !rtm->rtm_src_len) || 3146 (tb[RTA_DST] && !rtm->rtm_dst_len)) { 3147 NL_SET_ERR_MSG(extack, "ipv4: rtm_src_len and rtm_dst_len must be 32 for IPv4"); 3148 return -EINVAL; 3149 } 3150 3151 for (i = 0; i <= RTA_MAX; i++) { 3152 if (!tb[i]) 3153 continue; 3154 3155 switch (i) { 3156 case RTA_IIF: 3157 case RTA_OIF: 3158 case RTA_SRC: 3159 case RTA_DST: 3160 case RTA_IP_PROTO: 3161 case RTA_SPORT: 3162 case RTA_DPORT: 3163 case RTA_MARK: 3164 case RTA_UID: 3165 break; 3166 default: 3167 NL_SET_ERR_MSG(extack, "ipv4: Unsupported attribute in route get request"); 3168 return -EINVAL; 3169 } 3170 } 3171 3172 return 0; 3173 } 3174 3175 static int inet_rtm_getroute(struct sk_buff *in_skb, struct nlmsghdr *nlh, 3176 struct netlink_ext_ack *extack) 3177 { 3178 struct net *net = sock_net(in_skb->sk); 3179 struct nlattr *tb[RTA_MAX+1]; 3180 u32 table_id = RT_TABLE_MAIN; 3181 __be16 sport = 0, dport = 0; 3182 struct fib_result res = {}; 3183 u8 ip_proto = IPPROTO_UDP; 3184 struct rtable *rt = NULL; 3185 struct sk_buff *skb; 3186 struct rtmsg *rtm; 3187 struct flowi4 fl4 = {}; 3188 __be32 dst = 0; 3189 __be32 src = 0; 3190 kuid_t uid; 3191 u32 iif; 3192 int err; 3193 int mark; 3194 3195 err = inet_rtm_valid_getroute_req(in_skb, nlh, tb, extack); 3196 if (err < 0) 3197 return err; 3198 3199 rtm = nlmsg_data(nlh); 3200 src = tb[RTA_SRC] ? nla_get_in_addr(tb[RTA_SRC]) : 0; 3201 dst = tb[RTA_DST] ? nla_get_in_addr(tb[RTA_DST]) : 0; 3202 iif = tb[RTA_IIF] ? nla_get_u32(tb[RTA_IIF]) : 0; 3203 mark = tb[RTA_MARK] ? nla_get_u32(tb[RTA_MARK]) : 0; 3204 if (tb[RTA_UID]) 3205 uid = make_kuid(current_user_ns(), nla_get_u32(tb[RTA_UID])); 3206 else 3207 uid = (iif ? INVALID_UID : current_uid()); 3208 3209 if (tb[RTA_IP_PROTO]) { 3210 err = rtm_getroute_parse_ip_proto(tb[RTA_IP_PROTO], 3211 &ip_proto, AF_INET, extack); 3212 if (err) 3213 return err; 3214 } 3215 3216 if (tb[RTA_SPORT]) 3217 sport = nla_get_be16(tb[RTA_SPORT]); 3218 3219 if (tb[RTA_DPORT]) 3220 dport = nla_get_be16(tb[RTA_DPORT]); 3221 3222 skb = inet_rtm_getroute_build_skb(src, dst, ip_proto, sport, dport); 3223 if (!skb) 3224 return -ENOBUFS; 3225 3226 fl4.daddr = dst; 3227 fl4.saddr = src; 3228 fl4.flowi4_tos = rtm->rtm_tos & IPTOS_RT_MASK; 3229 fl4.flowi4_oif = tb[RTA_OIF] ? nla_get_u32(tb[RTA_OIF]) : 0; 3230 fl4.flowi4_mark = mark; 3231 fl4.flowi4_uid = uid; 3232 if (sport) 3233 fl4.fl4_sport = sport; 3234 if (dport) 3235 fl4.fl4_dport = dport; 3236 fl4.flowi4_proto = ip_proto; 3237 3238 rcu_read_lock(); 3239 3240 if (iif) { 3241 struct net_device *dev; 3242 3243 dev = dev_get_by_index_rcu(net, iif); 3244 if (!dev) { 3245 err = -ENODEV; 3246 goto errout_rcu; 3247 } 3248 3249 fl4.flowi4_iif = iif; /* for rt_fill_info */ 3250 skb->dev = dev; 3251 skb->mark = mark; 3252 err = ip_route_input_rcu(skb, dst, src, 3253 rtm->rtm_tos & IPTOS_RT_MASK, dev, 3254 &res); 3255 3256 rt = skb_rtable(skb); 3257 if (err == 0 && rt->dst.error) 3258 err = -rt->dst.error; 3259 } else { 3260 fl4.flowi4_iif = LOOPBACK_IFINDEX; 3261 skb->dev = net->loopback_dev; 3262 rt = ip_route_output_key_hash_rcu(net, &fl4, &res, skb); 3263 err = 0; 3264 if (IS_ERR(rt)) 3265 err = PTR_ERR(rt); 3266 else 3267 skb_dst_set(skb, &rt->dst); 3268 } 3269 3270 if (err) 3271 goto errout_rcu; 3272 3273 if (rtm->rtm_flags & RTM_F_NOTIFY) 3274 rt->rt_flags |= RTCF_NOTIFY; 3275 3276 if (rtm->rtm_flags & RTM_F_LOOKUP_TABLE) 3277 table_id = res.table ? res.table->tb_id : 0; 3278 3279 /* reset skb for netlink reply msg */ 3280 skb_trim(skb, 0); 3281 skb_reset_network_header(skb); 3282 skb_reset_transport_header(skb); 3283 skb_reset_mac_header(skb); 3284 3285 if (rtm->rtm_flags & RTM_F_FIB_MATCH) { 3286 struct fib_rt_info fri; 3287 3288 if (!res.fi) { 3289 err = fib_props[res.type].error; 3290 if (!err) 3291 err = -EHOSTUNREACH; 3292 goto errout_rcu; 3293 } 3294 fri.fi = res.fi; 3295 fri.tb_id = table_id; 3296 fri.dst = res.prefix; 3297 fri.dst_len = res.prefixlen; 3298 fri.tos = fl4.flowi4_tos; 3299 fri.type = rt->rt_type; 3300 fri.offload = 0; 3301 fri.trap = 0; 3302 if (res.fa_head) { 3303 struct fib_alias *fa; 3304 3305 hlist_for_each_entry_rcu(fa, res.fa_head, fa_list) { 3306 u8 slen = 32 - fri.dst_len; 3307 3308 if (fa->fa_slen == slen && 3309 fa->tb_id == fri.tb_id && 3310 fa->fa_tos == fri.tos && 3311 fa->fa_info == res.fi && 3312 fa->fa_type == fri.type) { 3313 fri.offload = fa->offload; 3314 fri.trap = fa->trap; 3315 break; 3316 } 3317 } 3318 } 3319 err = fib_dump_info(skb, NETLINK_CB(in_skb).portid, 3320 nlh->nlmsg_seq, RTM_NEWROUTE, &fri, 0); 3321 } else { 3322 err = rt_fill_info(net, dst, src, rt, table_id, &fl4, skb, 3323 NETLINK_CB(in_skb).portid, 3324 nlh->nlmsg_seq, 0); 3325 } 3326 if (err < 0) 3327 goto errout_rcu; 3328 3329 rcu_read_unlock(); 3330 3331 err = rtnl_unicast(skb, net, NETLINK_CB(in_skb).portid); 3332 3333 errout_free: 3334 return err; 3335 errout_rcu: 3336 rcu_read_unlock(); 3337 kfree_skb(skb); 3338 goto errout_free; 3339 } 3340 3341 void ip_rt_multicast_event(struct in_device *in_dev) 3342 { 3343 rt_cache_flush(dev_net(in_dev->dev)); 3344 } 3345 3346 #ifdef CONFIG_SYSCTL 3347 static int ip_rt_gc_interval __read_mostly = 60 * HZ; 3348 static int ip_rt_gc_min_interval __read_mostly = HZ / 2; 3349 static int ip_rt_gc_elasticity __read_mostly = 8; 3350 static int ip_min_valid_pmtu __read_mostly = IPV4_MIN_MTU; 3351 3352 static int ipv4_sysctl_rtcache_flush(struct ctl_table *__ctl, int write, 3353 void *buffer, size_t *lenp, loff_t *ppos) 3354 { 3355 struct net *net = (struct net *)__ctl->extra1; 3356 3357 if (write) { 3358 rt_cache_flush(net); 3359 fnhe_genid_bump(net); 3360 return 0; 3361 } 3362 3363 return -EINVAL; 3364 } 3365 3366 static struct ctl_table ipv4_route_table[] = { 3367 { 3368 .procname = "gc_thresh", 3369 .data = &ipv4_dst_ops.gc_thresh, 3370 .maxlen = sizeof(int), 3371 .mode = 0644, 3372 .proc_handler = proc_dointvec, 3373 }, 3374 { 3375 .procname = "max_size", 3376 .data = &ip_rt_max_size, 3377 .maxlen = sizeof(int), 3378 .mode = 0644, 3379 .proc_handler = proc_dointvec, 3380 }, 3381 { 3382 /* Deprecated. Use gc_min_interval_ms */ 3383 3384 .procname = "gc_min_interval", 3385 .data = &ip_rt_gc_min_interval, 3386 .maxlen = sizeof(int), 3387 .mode = 0644, 3388 .proc_handler = proc_dointvec_jiffies, 3389 }, 3390 { 3391 .procname = "gc_min_interval_ms", 3392 .data = &ip_rt_gc_min_interval, 3393 .maxlen = sizeof(int), 3394 .mode = 0644, 3395 .proc_handler = proc_dointvec_ms_jiffies, 3396 }, 3397 { 3398 .procname = "gc_timeout", 3399 .data = &ip_rt_gc_timeout, 3400 .maxlen = sizeof(int), 3401 .mode = 0644, 3402 .proc_handler = proc_dointvec_jiffies, 3403 }, 3404 { 3405 .procname = "gc_interval", 3406 .data = &ip_rt_gc_interval, 3407 .maxlen = sizeof(int), 3408 .mode = 0644, 3409 .proc_handler = proc_dointvec_jiffies, 3410 }, 3411 { 3412 .procname = "redirect_load", 3413 .data = &ip_rt_redirect_load, 3414 .maxlen = sizeof(int), 3415 .mode = 0644, 3416 .proc_handler = proc_dointvec, 3417 }, 3418 { 3419 .procname = "redirect_number", 3420 .data = &ip_rt_redirect_number, 3421 .maxlen = sizeof(int), 3422 .mode = 0644, 3423 .proc_handler = proc_dointvec, 3424 }, 3425 { 3426 .procname = "redirect_silence", 3427 .data = &ip_rt_redirect_silence, 3428 .maxlen = sizeof(int), 3429 .mode = 0644, 3430 .proc_handler = proc_dointvec, 3431 }, 3432 { 3433 .procname = "error_cost", 3434 .data = &ip_rt_error_cost, 3435 .maxlen = sizeof(int), 3436 .mode = 0644, 3437 .proc_handler = proc_dointvec, 3438 }, 3439 { 3440 .procname = "error_burst", 3441 .data = &ip_rt_error_burst, 3442 .maxlen = sizeof(int), 3443 .mode = 0644, 3444 .proc_handler = proc_dointvec, 3445 }, 3446 { 3447 .procname = "gc_elasticity", 3448 .data = &ip_rt_gc_elasticity, 3449 .maxlen = sizeof(int), 3450 .mode = 0644, 3451 .proc_handler = proc_dointvec, 3452 }, 3453 { 3454 .procname = "mtu_expires", 3455 .data = &ip_rt_mtu_expires, 3456 .maxlen = sizeof(int), 3457 .mode = 0644, 3458 .proc_handler = proc_dointvec_jiffies, 3459 }, 3460 { 3461 .procname = "min_pmtu", 3462 .data = &ip_rt_min_pmtu, 3463 .maxlen = sizeof(int), 3464 .mode = 0644, 3465 .proc_handler = proc_dointvec_minmax, 3466 .extra1 = &ip_min_valid_pmtu, 3467 }, 3468 { 3469 .procname = "min_adv_mss", 3470 .data = &ip_rt_min_advmss, 3471 .maxlen = sizeof(int), 3472 .mode = 0644, 3473 .proc_handler = proc_dointvec, 3474 }, 3475 { } 3476 }; 3477 3478 static const char ipv4_route_flush_procname[] = "flush"; 3479 3480 static struct ctl_table ipv4_route_flush_table[] = { 3481 { 3482 .procname = ipv4_route_flush_procname, 3483 .maxlen = sizeof(int), 3484 .mode = 0200, 3485 .proc_handler = ipv4_sysctl_rtcache_flush, 3486 }, 3487 { }, 3488 }; 3489 3490 static __net_init int sysctl_route_net_init(struct net *net) 3491 { 3492 struct ctl_table *tbl; 3493 3494 tbl = ipv4_route_flush_table; 3495 if (!net_eq(net, &init_net)) { 3496 tbl = kmemdup(tbl, sizeof(ipv4_route_flush_table), GFP_KERNEL); 3497 if (!tbl) 3498 goto err_dup; 3499 3500 /* Don't export non-whitelisted sysctls to unprivileged users */ 3501 if (net->user_ns != &init_user_ns) { 3502 if (tbl[0].procname != ipv4_route_flush_procname) 3503 tbl[0].procname = NULL; 3504 } 3505 } 3506 tbl[0].extra1 = net; 3507 3508 net->ipv4.route_hdr = register_net_sysctl(net, "net/ipv4/route", tbl); 3509 if (!net->ipv4.route_hdr) 3510 goto err_reg; 3511 return 0; 3512 3513 err_reg: 3514 if (tbl != ipv4_route_flush_table) 3515 kfree(tbl); 3516 err_dup: 3517 return -ENOMEM; 3518 } 3519 3520 static __net_exit void sysctl_route_net_exit(struct net *net) 3521 { 3522 struct ctl_table *tbl; 3523 3524 tbl = net->ipv4.route_hdr->ctl_table_arg; 3525 unregister_net_sysctl_table(net->ipv4.route_hdr); 3526 BUG_ON(tbl == ipv4_route_flush_table); 3527 kfree(tbl); 3528 } 3529 3530 static __net_initdata struct pernet_operations sysctl_route_ops = { 3531 .init = sysctl_route_net_init, 3532 .exit = sysctl_route_net_exit, 3533 }; 3534 #endif 3535 3536 static __net_init int rt_genid_init(struct net *net) 3537 { 3538 atomic_set(&net->ipv4.rt_genid, 0); 3539 atomic_set(&net->fnhe_genid, 0); 3540 atomic_set(&net->ipv4.dev_addr_genid, get_random_int()); 3541 return 0; 3542 } 3543 3544 static __net_initdata struct pernet_operations rt_genid_ops = { 3545 .init = rt_genid_init, 3546 }; 3547 3548 static int __net_init ipv4_inetpeer_init(struct net *net) 3549 { 3550 struct inet_peer_base *bp = kmalloc(sizeof(*bp), GFP_KERNEL); 3551 3552 if (!bp) 3553 return -ENOMEM; 3554 inet_peer_base_init(bp); 3555 net->ipv4.peers = bp; 3556 return 0; 3557 } 3558 3559 static void __net_exit ipv4_inetpeer_exit(struct net *net) 3560 { 3561 struct inet_peer_base *bp = net->ipv4.peers; 3562 3563 net->ipv4.peers = NULL; 3564 inetpeer_invalidate_tree(bp); 3565 kfree(bp); 3566 } 3567 3568 static __net_initdata struct pernet_operations ipv4_inetpeer_ops = { 3569 .init = ipv4_inetpeer_init, 3570 .exit = ipv4_inetpeer_exit, 3571 }; 3572 3573 #ifdef CONFIG_IP_ROUTE_CLASSID 3574 struct ip_rt_acct __percpu *ip_rt_acct __read_mostly; 3575 #endif /* CONFIG_IP_ROUTE_CLASSID */ 3576 3577 int __init ip_rt_init(void) 3578 { 3579 int cpu; 3580 3581 ip_idents = kmalloc_array(IP_IDENTS_SZ, sizeof(*ip_idents), 3582 GFP_KERNEL); 3583 if (!ip_idents) 3584 panic("IP: failed to allocate ip_idents\n"); 3585 3586 prandom_bytes(ip_idents, IP_IDENTS_SZ * sizeof(*ip_idents)); 3587 3588 ip_tstamps = kcalloc(IP_IDENTS_SZ, sizeof(*ip_tstamps), GFP_KERNEL); 3589 if (!ip_tstamps) 3590 panic("IP: failed to allocate ip_tstamps\n"); 3591 3592 for_each_possible_cpu(cpu) { 3593 struct uncached_list *ul = &per_cpu(rt_uncached_list, cpu); 3594 3595 INIT_LIST_HEAD(&ul->head); 3596 spin_lock_init(&ul->lock); 3597 } 3598 #ifdef CONFIG_IP_ROUTE_CLASSID 3599 ip_rt_acct = __alloc_percpu(256 * sizeof(struct ip_rt_acct), __alignof__(struct ip_rt_acct)); 3600 if (!ip_rt_acct) 3601 panic("IP: failed to allocate ip_rt_acct\n"); 3602 #endif 3603 3604 ipv4_dst_ops.kmem_cachep = 3605 kmem_cache_create("ip_dst_cache", sizeof(struct rtable), 0, 3606 SLAB_HWCACHE_ALIGN|SLAB_PANIC, NULL); 3607 3608 ipv4_dst_blackhole_ops.kmem_cachep = ipv4_dst_ops.kmem_cachep; 3609 3610 if (dst_entries_init(&ipv4_dst_ops) < 0) 3611 panic("IP: failed to allocate ipv4_dst_ops counter\n"); 3612 3613 if (dst_entries_init(&ipv4_dst_blackhole_ops) < 0) 3614 panic("IP: failed to allocate ipv4_dst_blackhole_ops counter\n"); 3615 3616 ipv4_dst_ops.gc_thresh = ~0; 3617 ip_rt_max_size = INT_MAX; 3618 3619 devinet_init(); 3620 ip_fib_init(); 3621 3622 if (ip_rt_proc_init()) 3623 pr_err("Unable to create route proc files\n"); 3624 #ifdef CONFIG_XFRM 3625 xfrm_init(); 3626 xfrm4_init(); 3627 #endif 3628 rtnl_register(PF_INET, RTM_GETROUTE, inet_rtm_getroute, NULL, 3629 RTNL_FLAG_DOIT_UNLOCKED); 3630 3631 #ifdef CONFIG_SYSCTL 3632 register_pernet_subsys(&sysctl_route_ops); 3633 #endif 3634 register_pernet_subsys(&rt_genid_ops); 3635 register_pernet_subsys(&ipv4_inetpeer_ops); 3636 return 0; 3637 } 3638 3639 #ifdef CONFIG_SYSCTL 3640 /* 3641 * We really need to sanitize the damn ipv4 init order, then all 3642 * this nonsense will go away. 3643 */ 3644 void __init ip_static_sysctl_init(void) 3645 { 3646 register_net_sysctl(&init_net, "net/ipv4/route", ipv4_route_table); 3647 } 3648 #endif 3649